feat: replace Subject retirement with durable explicit deletion
This commit is contained in:
@@ -1193,8 +1193,40 @@ pub struct SubjectCreateRecoveryResponse {
|
||||
pub operation_id: String,
|
||||
}
|
||||
|
||||
/// Host requests this only after durably fencing the Subject in Backend authority.
|
||||
/// Permanently exclude the Worker from creation and restoration, and confirm actual
|
||||
/// execution shutdown. This does not delete Worker or Session records.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjectDeleteWorkerRequest {
|
||||
pub workspace_id: String,
|
||||
pub worker_id: WorkerId,
|
||||
pub create_fingerprint: String,
|
||||
pub operation_id: String,
|
||||
}
|
||||
|
||||
/// Durable admission exclusion plus confirmed execution shutdown/absence. Errors,
|
||||
/// timeouts and lost responses are unknown outcomes; retry the same full request.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjectDeleteWorkerResponse {
|
||||
pub workspace_id: String,
|
||||
pub worker_id: WorkerId,
|
||||
pub create_fingerprint: String,
|
||||
pub operation_id: String,
|
||||
/// The canonical live Worker/Session/diagnostic aggregate is physically absent
|
||||
/// under the permanent creation fence. False is not a deletion receipt.
|
||||
pub originals_absent: bool,
|
||||
}
|
||||
|
||||
#[api(reqwest, axum)]
|
||||
pub trait RuntimeApi {
|
||||
#[post("/v1/subject-delete-worker", status = 200, error_status = 400)]
|
||||
async fn subject_delete_worker(
|
||||
&self,
|
||||
#[body] request: SubjectDeleteWorkerRequest,
|
||||
) -> Result<SubjectDeleteWorkerResponse, RuntimeApiError>;
|
||||
|
||||
#[post("/v1/subject-create-recovery", status = 200, error_status = 400)]
|
||||
async fn recover_subject_create(
|
||||
&self,
|
||||
@@ -1755,6 +1787,19 @@ mod tests {
|
||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
async fn subject_delete_worker(
|
||||
&self,
|
||||
request: SubjectDeleteWorkerRequest,
|
||||
) -> Result<SubjectDeleteWorkerResponse, RuntimeApiError> {
|
||||
Ok(SubjectDeleteWorkerResponse {
|
||||
originals_absent: true,
|
||||
workspace_id: request.workspace_id,
|
||||
worker_id: request.worker_id,
|
||||
create_fingerprint: request.create_fingerprint,
|
||||
operation_id: request.operation_id,
|
||||
})
|
||||
}
|
||||
|
||||
async fn recover_subject_create(
|
||||
&self,
|
||||
request: SubjectCreateRecoveryRequest,
|
||||
@@ -1934,6 +1979,19 @@ mod tests {
|
||||
assert_eq!(response.create_fingerprint, recovery.create_fingerprint);
|
||||
assert_eq!(response.operation_id, recovery.operation_id);
|
||||
|
||||
let deletion = SubjectDeleteWorkerRequest {
|
||||
workspace_id: "workspace-a".into(),
|
||||
worker_id: WorkerId::now_v7(),
|
||||
create_fingerprint: "deleted-create".into(),
|
||||
operation_id: "delete-1".into(),
|
||||
};
|
||||
let receipt = client
|
||||
.subject_delete_worker(deletion.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut expected = serde_json::to_value(deletion).unwrap();
|
||||
expected["originals_absent"] = true.into();
|
||||
assert_eq!(serde_json::to_value(receipt).unwrap(), expected);
|
||||
let summary = client.runtime_summary().await.unwrap();
|
||||
assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip"));
|
||||
let workers = client
|
||||
|
||||
@@ -338,6 +338,8 @@ impl_openapi_schema!(
|
||||
MemoryConsolidationResponse,
|
||||
SubjektivSubjectCreateRequest,
|
||||
SubjektivSubjectBehaviorUpdateRequest,
|
||||
SubjektivSubjectDeleteRequest,
|
||||
SubjektivSubjectDeleteResponse,
|
||||
SubjektivSubjectListQuery,
|
||||
SubjektivSubjectListResponse,
|
||||
SubjektivSubjectResponse,
|
||||
@@ -1527,19 +1529,21 @@ pub trait ServerApi {
|
||||
#[path] memory_id: String,
|
||||
#[query] query: SubjektivMemoryChangesQuery,
|
||||
) -> Result<SubjektivMemoryListChangesResponse, RepositoryApiError>;
|
||||
#[post(
|
||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire",
|
||||
#[delete(
|
||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}",
|
||||
status = 200,
|
||||
error_status = 400,
|
||||
additional_error_statuses = [401, 403, 404, 409, 500],
|
||||
bearer_auth = true,
|
||||
browser_auth = true,
|
||||
)]
|
||||
async fn subjektiv_subject_retire(
|
||||
async fn subjektiv_subject_delete(
|
||||
&self,
|
||||
#[extension] context: ServerRequestContext,
|
||||
#[path] workspace_id: String,
|
||||
#[path] subject_id: String,
|
||||
) -> Result<SubjektivSubjectResponse, RepositoryApiError>;
|
||||
#[body] request: SubjektivSubjectDeleteRequest,
|
||||
) -> Result<SubjektivSubjectDeleteResponse, RepositoryApiError>;
|
||||
#[post(
|
||||
"/api/w/{workspace_id}/subjektiv/staging",
|
||||
status = 200,
|
||||
@@ -4955,6 +4959,8 @@ pub struct WorkspacePermissionSummary {
|
||||
pub manage_secrets: bool,
|
||||
pub manage_runtimes: bool,
|
||||
pub delete_workspace: bool,
|
||||
/// Explicit Subject deletion authority for the current authenticated actor.
|
||||
pub delete_subjects: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
@@ -5350,7 +5356,25 @@ pub struct MemoryConsolidationResponse {
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SubjektivSubjectState {
|
||||
Active,
|
||||
Retired,
|
||||
/// Saved compatibility only; not a current lifecycle operation.
|
||||
#[serde(rename = "retired")]
|
||||
LegacyRetired,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjektivSubjectDeleteRequest {
|
||||
/// Stable identity retained for every retry, including unknown outcomes.
|
||||
pub operation_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjektivSubjectDeleteResponse {
|
||||
pub subject_id: String,
|
||||
pub operation_id: String,
|
||||
/// False confirms the deletion fence, not external stop or erasure completion.
|
||||
pub completed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
@@ -10950,6 +10974,10 @@ pub fn memory_api_typescript() -> String {
|
||||
|
||||
export type SubjektivSubjectCreateRequest = { role: string, behavior_md?: string, };
|
||||
|
||||
export type SubjektivSubjectDeleteRequest = { operation_id: string, };
|
||||
|
||||
export type SubjektivSubjectDeleteResponse = { subject_id: string, operation_id: string, completed: boolean, };
|
||||
|
||||
export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_md: string, behavior_md: string, };
|
||||
|
||||
export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, state: SubjektivSubjectState, memory_fingerprint: string, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, };
|
||||
@@ -13483,7 +13511,8 @@ mod tests {
|
||||
"manage_repositories": true,
|
||||
"manage_secrets": true,
|
||||
"manage_runtimes": true,
|
||||
"delete_workspace": true
|
||||
"delete_workspace": true,
|
||||
"delete_subjects": true
|
||||
},
|
||||
"extension_points": {
|
||||
"store": "sqlite",
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
use serde_json::json;
|
||||
use server_api::{
|
||||
SubjektivSubjectDeleteRequest, SubjektivSubjectDeleteResponse, SubjektivSubjectState,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn subject_delete_api_replaces_retire_and_requires_operation_identity() {
|
||||
let document: serde_json::Value = serde_json::from_str(
|
||||
&server_api::canonical_openapi_document()
|
||||
.unwrap()
|
||||
.to_json()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let paths = &document["paths"];
|
||||
assert!(
|
||||
paths
|
||||
.get("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire")
|
||||
.is_none()
|
||||
);
|
||||
let operation = &paths["/api/w/{workspace_id}/subjektiv/subjects/{subject_id}"]["delete"];
|
||||
assert_eq!(operation["operationId"], "subjektiv_subject_delete");
|
||||
assert_eq!(operation["requestBody"]["required"], true);
|
||||
assert!(serde_json::from_value::<SubjektivSubjectDeleteRequest>(json!({})).is_err());
|
||||
assert!(
|
||||
serde_json::from_value::<SubjektivSubjectDeleteRequest>(
|
||||
json!({"operation_id":"op", "force":true})
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
for completed in [false, true] {
|
||||
let response = SubjektivSubjectDeleteResponse {
|
||||
subject_id: "subject".into(),
|
||||
operation_id: "op".into(),
|
||||
completed,
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).unwrap(),
|
||||
json!({"subject_id":"subject", "operation_id":"op", "completed":completed})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_subject_state_is_explicit_saved_compatibility_not_a_new_operation() {
|
||||
let state: SubjektivSubjectState = serde_json::from_value(json!("retired")).unwrap();
|
||||
assert_eq!(state, SubjektivSubjectState::LegacyRetired);
|
||||
assert_eq!(serde_json::to_value(state).unwrap(), json!("retired"));
|
||||
}
|
||||
@@ -12,3 +12,49 @@ Hosts retain body ownership across inactivity, reserve actual execution quiescen
|
||||
- [`api`]: shared Host operations and bounded responses
|
||||
- [`job`]: consolidation selection rules and result validation
|
||||
- [`journal`]: individually committed activity operations, source coverage, retry receipts, and bounded reading
|
||||
|
||||
## Explicit Subject deletion
|
||||
|
||||
`begin_subject_deletion(subject_id, operation_id)` commits a permanent content-free
|
||||
fence in this store **before** the Host stops execution or erases external content.
|
||||
The operation ID is unique within this storage scope. Exact retries return current
|
||||
saved state; a different key for that Subject or reuse for another Subject conflicts.
|
||||
Pending fences survive restart and a failed purge. Normal reads, writes, saved-receipt
|
||||
replay, cached Host/Journal contexts and authority lookup cannot use a fenced Subject.
|
||||
Subject discovery hides fenced records before pagination. SQL INSERT/UPDATE guards
|
||||
cover direct and indirect owners, checking both OLD and NEW ownership on UPDATE.
|
||||
Deleting a grant never restores non-Subject fallback: content-free Worker identity
|
||||
fences remain, and the Host must consult them before fallback authorization.
|
||||
|
||||
The Host uses `subject_deletion` / `pending_subject_deletions` for recovery and
|
||||
`deletion_subject_workers` / `deletion_subject_sessions` / `deletion_subject_jobs`
|
||||
only as cleanup inventories, not authority or model-visible content. It must fence
|
||||
creation reservations, quiesce Workers and in-flight delivery, purge Job inputs/results
|
||||
and live Job execution copies, and clear resident/derived caches. Source Worker/Session
|
||||
originals and pre-existing archives have independent retention lifecycles: the Workspace
|
||||
Host archives source originals rather than erasing them. Those archives may contain
|
||||
previously injected Memory; Workdirs, Tickets, exports and external provider copies are
|
||||
also outside Subject content erasure. These boundaries must be explained to the user.
|
||||
The domain rechecks after out-of-DB capture callbacks and before returning Job read
|
||||
responses; already returned DTOs cannot be revoked by SQLite. Host lifecycle locking
|
||||
must cover delivery/reinjection of previously fetched content as well as finish.
|
||||
|
||||
Only after external cleanup succeeds may the Host call
|
||||
`finish_subject_deletion(subject_id, operation_id)`. One transaction deletes Subject,
|
||||
Memory/history, staging/evidence, Surface snapshots/runs, permission sources/receipts,
|
||||
child grants, writer/session bindings, Journal content/generations/deliveries/receipts,
|
||||
links, and the Subject's `legacy_record_archive` rows. A purge error rolls back all
|
||||
those deletions without undoing the separately committed fence. Ordinary history
|
||||
retention and explicit Journal Session-erasure rules are unchanged for other Subjects.
|
||||
Retained records contain only deletion operation/time/completion and execution/session
|
||||
identities needed to prevent reuse; existing content-free Session-erasure markers and
|
||||
workspace-only migration metadata remain. Other Subjects and storage scopes are untouched.
|
||||
|
||||
The deletion migration adds fences without erasing or reactivating existing data. Old `retired`
|
||||
JSON/SQL records deserialize as `SubjectState::LegacyRetired` and remain readable
|
||||
until explicitly deleted. There is no current retirement API. Normal Worker deletion
|
||||
still does not implicitly delete Subject memory.
|
||||
|
||||
Completion means logical erasure from domain and Host-managed storage, **not** secure
|
||||
physical overwriting: SQLite free pages, WAL files, filesystem snapshots, backups and
|
||||
external copies can retain bytes under their own retention/erasure policies.
|
||||
|
||||
@@ -116,6 +116,7 @@ impl HostOperationContext {
|
||||
"Host operation context belongs to another storage scope".into(),
|
||||
));
|
||||
}
|
||||
Self::require_subject(store, &self.subject_id)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -452,3 +452,45 @@ fn query_content_condition_ignores_behavior_and_detects_aba_memory_writes() {
|
||||
Err(OperationError::Conflict(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_revalidates_cached_body_empty_batch_and_receipt_contexts() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, store) = open(temp.path(), "scope");
|
||||
let subject = store
|
||||
.create_subject(SubjectRole::new("companion").unwrap())
|
||||
.unwrap();
|
||||
let body = HostOperationContext::validated_body(&store, &subject.id, None).unwrap();
|
||||
let empty =
|
||||
HostOperationContext::validated_consolidation(&store, &subject.id, Some(vec![]), None)
|
||||
.unwrap();
|
||||
store.begin_subject_deletion(&subject.id, "delete").unwrap();
|
||||
for context in [&body, &empty] {
|
||||
for operation in [
|
||||
query(None),
|
||||
Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None }),
|
||||
Op::ReceiptStatus(server_api::SubjektivMemoryReceiptStatusRequest {
|
||||
receipt_id: "receipt".into(),
|
||||
}),
|
||||
Op::ReadCandidate(server_api::SubjektivMemoryCandidateReadRequest {
|
||||
candidate_id: "candidate".into(),
|
||||
}),
|
||||
] {
|
||||
assert!(matches!(
|
||||
execute(&store, context, operation),
|
||||
Err(OperationError::SubjectNotFound(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
store
|
||||
.finish_subject_deletion(&subject.id, "delete")
|
||||
.unwrap();
|
||||
assert!(
|
||||
execute(
|
||||
&store,
|
||||
&empty,
|
||||
Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None })
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
//! Explicit Subject erasure is a two-phase Host operation. Beginning commits a permanent,
|
||||
//! content-free fence before the Host stops/removes Workers and purges Jobs and execution copies.
|
||||
//! Source Worker/Session originals remain under their separate archive lifecycle.
|
||||
//! Failure leaves that fence and the cleanup inventory intact; the Host retries
|
||||
//! the same operation ID. Finishing asserts that external cleanup succeeded and atomically erases
|
||||
//! this store's content. This store cannot verify external Runtime/Job/Session cleanup itself.
|
||||
//!
|
||||
//! Tombstones contain no behavior, permission vectors, Memory, Journal or Session text. Worker
|
||||
//! identities remain permanently fenced so removal of a grant cannot become non-Subject fallback
|
||||
//! authority. Existing Journal Session erasure markers are retained to reject delayed admission;
|
||||
//! they are content-free and are not a second copy of the deleted material.
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjectDeletion {
|
||||
pub subject_id: String,
|
||||
pub operation_id: String,
|
||||
pub completed: bool,
|
||||
pub started_at: String,
|
||||
}
|
||||
|
||||
/// Host-only cleanup identity, never Job input, output, profile or authorization material.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DeletionSubjectJob {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
}
|
||||
|
||||
// All content tables are enumerated deliberately. Indirect tables are erased before their
|
||||
// owner rows, both for FK checks and so their fencing predicates remain resolvable.
|
||||
const INDIRECT: &[(&str, &str)] = &[
|
||||
(
|
||||
"journal_generation_input_deliveries",
|
||||
"generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})",
|
||||
),
|
||||
(
|
||||
"legacy_record_archive",
|
||||
"json_extract(row_json, '$.subject_id')={subject}",
|
||||
),
|
||||
(
|
||||
"journal_generation_deliveries",
|
||||
"generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})",
|
||||
),
|
||||
(
|
||||
"journal_operations",
|
||||
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
|
||||
),
|
||||
(
|
||||
"journal_generations",
|
||||
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
|
||||
),
|
||||
(
|
||||
"journal_sources",
|
||||
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
|
||||
),
|
||||
(
|
||||
"journal_session_bindings",
|
||||
"binding_id IN (SELECT binding_id FROM journal_writer_bindings WHERE subject_id={subject})",
|
||||
),
|
||||
];
|
||||
const DIRECT: &[&str] = &[
|
||||
"memory_journal_links",
|
||||
"journal_corrections",
|
||||
"journal_entries",
|
||||
"journals",
|
||||
"journal_writer_bindings",
|
||||
"subject_session_attributions",
|
||||
"worker_execution_grants",
|
||||
"subject_permission_receipts",
|
||||
"subject_permissions",
|
||||
"subject_permission_changes",
|
||||
"candidate_decision_receipts",
|
||||
"surface_generation_state",
|
||||
"surface_generation_runs",
|
||||
"surface_snapshot_refs",
|
||||
"surface_snapshot_seals",
|
||||
"surface_snapshots",
|
||||
"staging_resolution_targets",
|
||||
"staging_resolution_seals",
|
||||
"staging_resolutions",
|
||||
"memory_change_candidates",
|
||||
"memory_change_derivations",
|
||||
"memory_change_seals",
|
||||
"memory_changes",
|
||||
"memory_records",
|
||||
"staging_records",
|
||||
"subjects",
|
||||
];
|
||||
|
||||
fn owner_expression(table: &str, row: &str) -> String {
|
||||
match table {
|
||||
"legacy_record_archive" => format!("json_extract({row}.row_json, '$.subject_id')"),
|
||||
"journal_generation_input_deliveries" | "journal_generation_deliveries" => format!(
|
||||
"(SELECT j.subject_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE g.id={row}.generation_id)"
|
||||
),
|
||||
"journal_operations" | "journal_generations" | "journal_sources" => {
|
||||
format!("(SELECT subject_id FROM journals WHERE id={row}.journal_id)")
|
||||
}
|
||||
"journal_session_bindings" => format!(
|
||||
"(SELECT subject_id FROM journal_writer_bindings WHERE binding_id={row}.binding_id)"
|
||||
),
|
||||
_ => format!("{row}.subject_id"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn migrate(tx: &Transaction<'_>) -> feature_storage::Result<()> {
|
||||
tx.execute_batch(
|
||||
"CREATE TABLE subject_deletions (
|
||||
subject_id TEXT PRIMARY KEY,
|
||||
operation_id TEXT NOT NULL UNIQUE,
|
||||
completed INTEGER NOT NULL CHECK(completed IN (0,1)),
|
||||
started_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE subject_deleted_workers (
|
||||
subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
PRIMARY KEY(subject_id,runtime_id,worker_id)
|
||||
);
|
||||
CREATE TABLE subject_deleted_sessions (
|
||||
subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT,
|
||||
session_id TEXT NOT NULL UNIQUE,
|
||||
PRIMARY KEY(subject_id,session_id)
|
||||
);
|
||||
CREATE TRIGGER subject_deleted_sessions_retained BEFORE DELETE ON subject_deleted_sessions
|
||||
BEGIN SELECT RAISE(ABORT, 'deleted Session fences are permanent'); END;
|
||||
CREATE TRIGGER subject_deleted_sessions_immutable BEFORE UPDATE ON subject_deleted_sessions
|
||||
BEGIN SELECT RAISE(ABORT, 'deleted Session fences are immutable'); END;
|
||||
CREATE INDEX subject_deleted_worker_identity ON subject_deleted_workers(runtime_id,worker_id);
|
||||
CREATE TRIGGER subject_deletions_retained BEFORE DELETE ON subject_deletions
|
||||
BEGIN SELECT RAISE(ABORT, 'Subject deletion fences are permanent'); END;
|
||||
CREATE TRIGGER subject_deletions_identity BEFORE UPDATE ON subject_deletions
|
||||
WHEN OLD.subject_id<>NEW.subject_id OR OLD.operation_id<>NEW.operation_id
|
||||
OR OLD.started_at<>NEW.started_at OR OLD.completed>NEW.completed
|
||||
BEGIN SELECT RAISE(ABORT, 'Subject deletion identity is permanent'); END;
|
||||
CREATE TRIGGER subject_deleted_workers_retained BEFORE DELETE ON subject_deleted_workers
|
||||
BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are permanent'); END;
|
||||
CREATE TRIGGER subject_deleted_workers_immutable BEFORE UPDATE ON subject_deleted_workers
|
||||
BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are immutable'); END;",
|
||||
)?;
|
||||
|
||||
// Preserve each existing history-retention condition, adding only the explicit Subject
|
||||
// erasure exception. Frozen earlier migrations are not edited. Normal history deletion
|
||||
// (including another Subject's history) remains forbidden.
|
||||
let triggers = tx
|
||||
.prepare("SELECT name,tbl_name,sql FROM sqlite_master WHERE type='trigger'")?
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, String>(2)?,
|
||||
))
|
||||
})?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
for (name, table, sql) in triggers {
|
||||
if !DIRECT.contains(&table.as_str()) && !INDIRECT.iter().any(|(t, _)| *t == table) {
|
||||
continue;
|
||||
}
|
||||
let upper = sql.to_ascii_uppercase();
|
||||
if !upper.contains("BEFORE DELETE") {
|
||||
continue;
|
||||
}
|
||||
let begin = upper.find("BEGIN").expect("trigger body");
|
||||
let (header, body) = sql.split_at(begin);
|
||||
let exception = format!(
|
||||
"NOT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id={})",
|
||||
owner_expression(&table, "OLD")
|
||||
);
|
||||
let header = if let Some(when) = header.to_ascii_uppercase().find("WHEN") {
|
||||
format!(
|
||||
"{} WHEN ({}) AND {exception} ",
|
||||
&header[..when],
|
||||
header[when + 4..].trim()
|
||||
)
|
||||
} else {
|
||||
format!("{header} WHEN {exception} ")
|
||||
};
|
||||
tx.execute_batch(&format!("DROP TRIGGER \"{name}\"; {header}{body}"))?;
|
||||
}
|
||||
for table in DIRECT
|
||||
.iter()
|
||||
.copied()
|
||||
.chain(INDIRECT.iter().map(|(t, _)| *t))
|
||||
.chain(["journal_erased_sessions"])
|
||||
{
|
||||
for action in ["INSERT", "UPDATE"] {
|
||||
let new_owner = owner_expression(table, "NEW");
|
||||
let predicate = if action == "UPDATE" {
|
||||
format!(
|
||||
"subject_id={new_owner} OR subject_id={}",
|
||||
owner_expression(table, "OLD")
|
||||
)
|
||||
} else {
|
||||
format!("subject_id={new_owner}")
|
||||
};
|
||||
tx.execute_batch(&format!(
|
||||
"CREATE TRIGGER deletion_fence_{table}_{action} BEFORE {action} ON {table}
|
||||
WHEN EXISTS(SELECT 1 FROM subject_deletions WHERE {predicate})
|
||||
BEGIN SELECT RAISE(ABORT, 'Subject deletion has begun'); END;"
|
||||
))?;
|
||||
}
|
||||
}
|
||||
// Identity reuse must not reconnect purged content or turn a deleted child into
|
||||
// an unscoped Worker. Check both old and new on ownership-changing UPDATEs.
|
||||
for table in [
|
||||
"subject_session_attributions",
|
||||
"journal_writer_bindings",
|
||||
"worker_execution_grants",
|
||||
] {
|
||||
for action in ["INSERT", "UPDATE"] {
|
||||
let rows = if action == "UPDATE" {
|
||||
vec!["OLD", "NEW"]
|
||||
} else {
|
||||
vec!["NEW"]
|
||||
};
|
||||
let mut predicates = Vec::new();
|
||||
for row in rows {
|
||||
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id={row}.runtime_id AND worker_id={row}.worker_id)"));
|
||||
if table == "subject_session_attributions" {
|
||||
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_sessions WHERE session_id={row}.session_id)"));
|
||||
}
|
||||
if table == "worker_execution_grants" {
|
||||
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=json_extract({row}.record_json,'$.origin_runtime_id') AND worker_id=json_extract({row}.record_json,'$.origin_worker_id'))"));
|
||||
}
|
||||
}
|
||||
tx.execute_batch(&format!("CREATE TRIGGER deletion_identity_{table}_{action} BEFORE {action} ON {table} WHEN {} BEGIN SELECT RAISE(ABORT, 'deleted Subject execution identity'); END;", predicates.join(" OR ")))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn is_deleting(connection: &Connection, subject: &str) -> Result<bool> {
|
||||
// Older migration fixtures deliberately use current Rust reads before upgrade. Production
|
||||
// opens only registered current stores and must fail closed if the fence table is absent.
|
||||
#[cfg(test)]
|
||||
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions' AND type='table')", [], |r|r.get::<_,bool>(0))? {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(connection.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id=?1)",
|
||||
[subject],
|
||||
|r| r.get::<_, bool>(0),
|
||||
)?)
|
||||
}
|
||||
|
||||
pub(crate) fn require_not_deleting(connection: &Connection, subject: &str) -> Result<()> {
|
||||
if is_deleting(connection, subject)? {
|
||||
return Err(SubjektivError::SubjectNotFound(subject.into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(crate) fn require_live_worker(
|
||||
connection: &Connection,
|
||||
runtime: &str,
|
||||
worker: &str,
|
||||
) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_workers' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(()); }
|
||||
if connection.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)",
|
||||
params![runtime, worker],
|
||||
|r| r.get::<_, bool>(0),
|
||||
)? {
|
||||
return Err(SubjektivError::AuthorizationDenied(
|
||||
"Worker belongs to a deleted Subject".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn deleted_session_subject(
|
||||
connection: &Connection,
|
||||
session: &str,
|
||||
) -> Result<Option<String>> {
|
||||
#[cfg(test)]
|
||||
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_sessions' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(None); }
|
||||
Ok(connection
|
||||
.query_row(
|
||||
"SELECT subject_id FROM subject_deleted_sessions WHERE session_id=?1",
|
||||
[session],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
fn deletion(connection: &Connection, subject: &str) -> Result<Option<SubjectDeletion>> {
|
||||
Ok(connection.query_row("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE subject_id=?1", [subject], |r|Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? })).optional()?)
|
||||
}
|
||||
fn require_deletion(connection: &Connection, subject: &str) -> Result<SubjectDeletion> {
|
||||
deletion(connection, subject)?.ok_or_else(|| SubjektivError::SubjectNotFound(subject.into()))
|
||||
}
|
||||
|
||||
impl SubjektivStore {
|
||||
/// Commits the read/write/authority fence, even for legacy-retired Subjects. Operation IDs
|
||||
/// are unique within this store's scope; neither another key nor another Subject may reuse
|
||||
/// a deletion. Exact retries return the saved current result, including completed deletions.
|
||||
pub fn begin_subject_deletion(
|
||||
&self,
|
||||
subject: &str,
|
||||
operation: &str,
|
||||
) -> Result<SubjectDeletion> {
|
||||
validate_label("subject id", subject)?;
|
||||
validate_label("deletion operation id", operation)?;
|
||||
self.database.try_transaction(|tx| {
|
||||
if let Some(saved) = deletion(tx,subject)? {
|
||||
return if saved.operation_id == operation { Ok(saved) } else { Err(SubjektivError::SubjectDeletionConflict(operation.into())) };
|
||||
}
|
||||
if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE operation_id=?1)",[operation],|r|r.get::<_,bool>(0))? {
|
||||
return Err(SubjektivError::SubjectDeletionConflict(operation.into()));
|
||||
}
|
||||
require_subject(tx,subject)?;
|
||||
tx.execute("INSERT INTO subject_deletions(subject_id,operation_id,completed,started_at) VALUES(?1,?2,0,?3)",params![subject,operation,now()])?;
|
||||
// Capture bodies, historical attribution, delegated children, and grant origins.
|
||||
// This transaction also fences delayed grants/attributions, so no new identity can
|
||||
// appear after the inventory is frozen. Runtime create reservations remain Host-owned.
|
||||
tx.execute("INSERT INTO subject_deleted_workers(subject_id,runtime_id,worker_id)
|
||||
SELECT subject_id,runtime_id,worker_id FROM subject_session_attributions WHERE subject_id=?1
|
||||
UNION SELECT subject_id,runtime_id,worker_id FROM journal_writer_bindings WHERE subject_id=?1
|
||||
UNION SELECT subject_id,runtime_id,worker_id FROM worker_execution_grants WHERE subject_id=?1
|
||||
UNION SELECT subject_id,json_extract(record_json,'$.origin_runtime_id'),json_extract(record_json,'$.origin_worker_id') FROM worker_execution_grants WHERE subject_id=?1",[subject])?;
|
||||
tx.execute("INSERT INTO subject_deleted_sessions(subject_id,session_id)
|
||||
SELECT subject_id,session_id FROM subject_session_attributions WHERE subject_id=?1
|
||||
UNION SELECT subject_id,source_session_id FROM journals WHERE subject_id=?1
|
||||
UNION SELECT subject_id,session_id FROM journal_erased_sessions WHERE subject_id=?1", [subject])?;
|
||||
require_deletion(tx,subject)
|
||||
})
|
||||
}
|
||||
|
||||
/// Host-only, content-free cleanup inventory. Usable after begin, including exact finish
|
||||
/// retries. Tombstones survive finish to prevent non-Subject permission fallback.
|
||||
pub fn deletion_subject_workers(&self, subject: &str) -> Result<Vec<(String, String)>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
require_deletion(tx,subject)?;
|
||||
Ok(tx.prepare("SELECT runtime_id,worker_id FROM subject_deleted_workers WHERE subject_id=?1 ORDER BY runtime_id,worker_id")?.query_map([subject],|r|Ok((r.get(0)?,r.get(1)?)))?.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||
})
|
||||
}
|
||||
|
||||
/// Host-only Session identities; retained source rows are never exposed through this path.
|
||||
/// The Host must erase originals/archives before finish removes attribution.
|
||||
pub fn deletion_subject_sessions(&self, subject: &str) -> Result<Vec<String>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
require_deletion(tx,subject)?;
|
||||
Ok(tx.prepare("SELECT session_id FROM subject_deleted_sessions WHERE subject_id=?1 ORDER BY session_id")?.query_map([subject],|r|r.get(0))?.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||
})
|
||||
}
|
||||
|
||||
/// Host-only Job cleanup inventory. Job storage owns inputs, outcomes and Worker archives;
|
||||
/// this store retains only these identities until Host cleanup has succeeded.
|
||||
pub fn deletion_subject_jobs(&self, subject: &str) -> Result<Vec<DeletionSubjectJob>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
require_deletion(tx,subject)?;
|
||||
Ok(tx.prepare("SELECT job_id,attempt_id FROM surface_generation_runs WHERE subject_id=?1 AND job_id IS NOT NULL AND attempt_id IS NOT NULL UNION SELECT g.job_id,g.attempt_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id=?1 ORDER BY 1,2")?.query_map([subject],|r|Ok(DeletionSubjectJob {job_id:r.get(0)?,attempt_id:r.get(1)?}))?.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||
})
|
||||
}
|
||||
|
||||
/// Host-only recovery state; this never exposes Subject content or authority.
|
||||
pub fn subject_deletion(&self, subject: &str) -> Result<Option<SubjectDeletion>> {
|
||||
self.database
|
||||
.try_with_connection(|tx| deletion(tx, subject))
|
||||
}
|
||||
|
||||
/// Durable unfinished operations, including after Host restart. Completed tombstones
|
||||
/// remain queryable individually but are not returned as pending work.
|
||||
pub fn pending_subject_deletions(&self) -> Result<Vec<SubjectDeletion>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.prepare("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE completed=0 ORDER BY started_at,subject_id")?
|
||||
.query_map([], |r| Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? }))?
|
||||
.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn worker_subject_deleted(&self, runtime: &str, worker: &str) -> Result<bool> {
|
||||
self.database.try_with_connection(|tx| Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)",params![runtime,worker],|r|r.get(0))?))
|
||||
}
|
||||
|
||||
/// Host confirmation that external cleanup is complete. All Subject content and completed
|
||||
/// receipt commit together. Deferred FKs allow cyclic current/history references to disappear
|
||||
/// without weakening FK enforcement outside this transaction. Failure rolls back all erasure.
|
||||
pub fn finish_subject_deletion(
|
||||
&self,
|
||||
subject: &str,
|
||||
operation: &str,
|
||||
) -> Result<SubjectDeletion> {
|
||||
self.database.try_transaction(|tx| {
|
||||
let saved = require_deletion(tx, subject)?;
|
||||
if saved.operation_id != operation {
|
||||
return Err(SubjektivError::SubjectDeletionConflict(operation.into()));
|
||||
}
|
||||
if saved.completed {
|
||||
return Ok(saved);
|
||||
}
|
||||
tx.execute_batch("PRAGMA defer_foreign_keys=ON;")?;
|
||||
for (table, predicate) in INDIRECT {
|
||||
tx.execute(
|
||||
&format!(
|
||||
"DELETE FROM {table} WHERE {}",
|
||||
predicate.replace("{subject}", "?1")
|
||||
),
|
||||
[subject],
|
||||
)?;
|
||||
}
|
||||
for table in DIRECT {
|
||||
tx.execute(
|
||||
&format!("DELETE FROM {table} WHERE subject_id=?1"),
|
||||
[subject],
|
||||
)?;
|
||||
}
|
||||
tx.execute(
|
||||
"UPDATE subject_deletions SET completed=1 WHERE subject_id=?1",
|
||||
[subject],
|
||||
)?;
|
||||
require_deletion(tx, subject)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,489 @@
|
||||
use super::*;
|
||||
use crate::tests::{candidate, legacy_retired_fixture, open_store, role};
|
||||
use feature_storage::FeatureStorage;
|
||||
use memory::extract::CandidateKind;
|
||||
|
||||
fn draft() -> MemoryDraft {
|
||||
MemoryDraft::active(
|
||||
CandidateKind::Lesson,
|
||||
"private claim",
|
||||
"private body",
|
||||
"useful",
|
||||
"initial",
|
||||
)
|
||||
}
|
||||
|
||||
fn count(store: &SubjektivStore, table: &str) -> i64 {
|
||||
store
|
||||
.database
|
||||
.try_with_connection::<_, SubjektivError>(|c| {
|
||||
Ok(c.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0))?)
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fence_only_denies_content_reads_writes_and_exact_replay_before_purge() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
let subject = store
|
||||
.create_subject_with_behavior(role(), "private behavior".into())
|
||||
.unwrap();
|
||||
let id = &subject.id;
|
||||
let attribution = SubjectSessionAttribution::new(id, "runtime", "worker", "session").unwrap();
|
||||
store
|
||||
.record_session_attribution(attribution.clone())
|
||||
.unwrap();
|
||||
let staged = candidate(id, "candidate", store.workspace_id());
|
||||
store.stage_candidate(staged.clone()).unwrap();
|
||||
let request = CandidateDecisionRequest {
|
||||
request_id: "decision".into(),
|
||||
candidate_id: "candidate".into(),
|
||||
reason: "adopt".into(),
|
||||
decision: CandidateDecision::Apply {
|
||||
target: MemoryChangeTarget::Create,
|
||||
draft: draft(),
|
||||
},
|
||||
};
|
||||
let memory = store
|
||||
.decide_candidate(id, request.clone())
|
||||
.unwrap()
|
||||
.memory
|
||||
.unwrap();
|
||||
let generation = store
|
||||
.prepare_job_surface_generation(id, Some(("job", "attempt")))
|
||||
.unwrap();
|
||||
let snapshot = store
|
||||
.publish_surface_generation(
|
||||
id,
|
||||
&generation.id,
|
||||
vec![SurfacePoint {
|
||||
body_md: "private surface".into(),
|
||||
memory_refs: vec![MemoryChangeRef {
|
||||
memory_id: memory.id.clone(),
|
||||
change_id: memory.change_id.clone(),
|
||||
}],
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
let fence = store.begin_subject_deletion(id, "delete").unwrap();
|
||||
assert!(!fence.completed);
|
||||
// The fence must work while every body remains physically present.
|
||||
assert_eq!(count(&store, "subjects"), 1);
|
||||
assert_eq!(count(&store, "memory_records"), 1);
|
||||
assert_eq!(count(&store, "candidate_decision_receipts"), 1);
|
||||
assert!(store.subject(id).unwrap().is_none());
|
||||
assert!(store.list_subjects(1).unwrap().items.is_empty());
|
||||
macro_rules! denied { ($($result:expr),+ $(,)?) => { $(assert!($result.is_err(), stringify!($result));)+ }; }
|
||||
denied!(
|
||||
store.session_attribution("session"),
|
||||
store.subject_session_attributions(id),
|
||||
store.subject_session_attribution_page(id, None, "2999", None, 1),
|
||||
store.record_session_attribution(attribution),
|
||||
store.staging_candidate(id, "candidate"),
|
||||
store.pending_staging_candidates(id, 1),
|
||||
store.pending_staging_backlog(id),
|
||||
store.staging_resolution(id, "candidate"),
|
||||
store.memory(id, &memory.id),
|
||||
store.memory_change(id, &memory.id, memory.change_id.clone()),
|
||||
store.scoped_memory(id, &memory.id),
|
||||
store.scoped_memory_change(id, &memory.id, memory.change_id.clone()),
|
||||
store.list_memories(id),
|
||||
store.list_memory_changes(id, &memory.id),
|
||||
store.memory_fingerprint(id),
|
||||
store.surface_snapshot(id, &snapshot.id),
|
||||
store.resident_surface(id),
|
||||
store.resident_context(id),
|
||||
store.require_job_surface_generation(id, &generation.id, "job", "attempt"),
|
||||
store.validate_job_surface_outcome(
|
||||
id,
|
||||
&generation.id,
|
||||
generation.memory_fingerprint,
|
||||
"ready",
|
||||
Some(&snapshot.id),
|
||||
None,
|
||||
"job",
|
||||
"attempt"
|
||||
),
|
||||
store.prepare_surface_generation(id),
|
||||
store.publish_surface_generation(id, &generation.id, vec![]),
|
||||
store.fail_surface_generation(id, &generation.id, "failed"),
|
||||
store.update_subject_behavior(id, &subject.behavior_md, "late behavior".into()),
|
||||
store.stage_candidate(staged),
|
||||
store.decide_candidate(id, request),
|
||||
store.create_memory(id, draft()),
|
||||
store.subject_permissions(id),
|
||||
store.list_subject_worker_execution_grants(id),
|
||||
);
|
||||
assert_eq!(
|
||||
store.deletion_subject_workers(id).unwrap(),
|
||||
vec![("runtime".into(), "worker".into())]
|
||||
);
|
||||
assert_eq!(
|
||||
store.deletion_subject_sessions(id).unwrap(),
|
||||
vec!["session"]
|
||||
);
|
||||
assert_eq!(store.deletion_subject_jobs(id).unwrap()[0].job_id, "job");
|
||||
assert!(store.worker_subject_deleted("runtime", "worker").unwrap());
|
||||
assert!(
|
||||
store
|
||||
.finish_subject_deletion(id, "delete")
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
for table in DIRECT {
|
||||
assert_eq!(count(&store, table), 0, "{table}");
|
||||
}
|
||||
assert!(store.worker_execution_grant("runtime", "worker").is_err());
|
||||
assert_eq!(
|
||||
store.deletion_subject_sessions(id).unwrap(),
|
||||
vec!["session"]
|
||||
);
|
||||
assert_eq!(
|
||||
store.finish_subject_deletion(id, "delete").unwrap(),
|
||||
store.begin_subject_deletion(id, "delete").unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purge_failure_rolls_back_content_but_not_the_committed_fence_and_restart_recovers() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let id: String;
|
||||
let pending;
|
||||
{
|
||||
let (manager, scope, store) = open_store(temp.path(), "workspace-a");
|
||||
id = store.create_subject(role()).unwrap().id;
|
||||
store.create_memory(&id, draft()).unwrap();
|
||||
pending = store.begin_subject_deletion(&id, "delete").unwrap();
|
||||
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||
c.execute_batch("CREATE TRIGGER injected_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT, 'simulated purge failure'); END;")?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert!(store.finish_subject_deletion(&id, "delete").is_err());
|
||||
assert_eq!(count(&store, "memory_records"), 1);
|
||||
assert_eq!(count(&store, "memory_changes"), 1);
|
||||
assert_eq!(store.subject_deletion(&id).unwrap(), Some(pending.clone()));
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
}
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
assert_eq!(
|
||||
store.pending_subject_deletions().unwrap(),
|
||||
vec![pending.clone()]
|
||||
);
|
||||
assert_eq!(
|
||||
store.begin_subject_deletion(&id, "delete").unwrap(),
|
||||
pending
|
||||
);
|
||||
assert!(store.list_memories(&id).is_err());
|
||||
store
|
||||
.database
|
||||
.try_with_connection::<_, SubjektivError>(|c| {
|
||||
c.execute_batch("DROP TRIGGER injected_failure")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.finish_subject_deletion(&id, "delete")
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
assert!(store.pending_subject_deletions().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_identity_conflicts_are_scope_wide_but_do_not_cross_workspaces() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
let a = store.create_subject(role()).unwrap().id;
|
||||
let b = store.create_subject(role()).unwrap().id;
|
||||
let saved = store.begin_subject_deletion(&a, "same-key").unwrap();
|
||||
assert_eq!(store.begin_subject_deletion(&a, "same-key").unwrap(), saved);
|
||||
for result in [
|
||||
store.begin_subject_deletion(&a, "different"),
|
||||
store.begin_subject_deletion(&b, "same-key"),
|
||||
store.finish_subject_deletion(&a, "different"),
|
||||
] {
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(SubjektivError::SubjectDeletionConflict(_))
|
||||
));
|
||||
}
|
||||
assert!(store.begin_subject_deletion("absent", "new-key").is_err());
|
||||
assert!(store.finish_subject_deletion(&b, "same-key").is_err());
|
||||
let page = store.list_subjects(1).unwrap();
|
||||
assert_eq!(page.items[0].id, b);
|
||||
assert!(!page.has_more);
|
||||
store.finish_subject_deletion(&a, "same-key").unwrap();
|
||||
store.create_memory(&b, draft()).unwrap();
|
||||
let (_other_manager, _other_scope, other) = open_store(temp.path(), "workspace-b");
|
||||
assert!(other.subject_deletion(&a).unwrap().is_none());
|
||||
assert!(other.begin_subject_deletion(&b, "same-key").is_err());
|
||||
let c = other.create_subject(role()).unwrap().id;
|
||||
other.begin_subject_deletion(&c, "same-key").unwrap();
|
||||
assert_eq!(store.list_memories(&b).unwrap().len(), 1);
|
||||
let fresh = store.create_subject(role()).unwrap().id;
|
||||
assert!(store.list_memories(&fresh).unwrap().is_empty());
|
||||
assert!(store.subject_permissions(&fresh).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_begin_and_write_serialize_and_exact_finish_is_replayable() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
let id = store.create_subject(role()).unwrap().id;
|
||||
let barrier = std::sync::Barrier::new(3);
|
||||
std::thread::scope(|scope| {
|
||||
let a = scope.spawn(|| {
|
||||
barrier.wait();
|
||||
store.begin_subject_deletion(&id, "delete")
|
||||
});
|
||||
let b = scope.spawn(|| {
|
||||
barrier.wait();
|
||||
store.create_memory(&id, draft())
|
||||
});
|
||||
barrier.wait();
|
||||
assert!(!a.join().unwrap().unwrap().completed);
|
||||
// A write either precedes begin and is later purged, or loses to the fence.
|
||||
let _ = b.join().unwrap();
|
||||
});
|
||||
assert!(store.create_memory(&id, draft()).is_err());
|
||||
assert_eq!(store.pending_subject_deletions().unwrap().len(), 1);
|
||||
store.finish_subject_deletion(&id, "delete").unwrap();
|
||||
assert_eq!(count(&store, "memory_records"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_subject_owned_table_has_old_and_new_sql_fences_and_retention_is_scoped() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
let a = store.create_subject(role()).unwrap().id;
|
||||
let b = store.create_subject(role()).unwrap().id;
|
||||
store
|
||||
.stage_candidate(candidate(&a, "a", store.workspace_id()))
|
||||
.unwrap();
|
||||
store
|
||||
.stage_candidate(candidate(&b, "b", store.workspace_id()))
|
||||
.unwrap();
|
||||
store.begin_subject_deletion(&a, "delete").unwrap();
|
||||
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||
let tables = c.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")?.query_map([], |r| r.get::<_,String>(0))?.collect::<std::result::Result<Vec<_>,_>>()?;
|
||||
for table in tables {
|
||||
let columns = c.prepare(&format!("PRAGMA table_info({table})"))?.query_map([], |r| r.get::<_,String>(1))?.collect::<std::result::Result<Vec<_>,_>>()?;
|
||||
if !columns.iter().any(|v| v == "subject_id") || table.starts_with("subject_delet") { continue; }
|
||||
assert!(DIRECT.contains(&table.as_str()) || table == "journal_erased_sessions", "unclassified Subject table: {table}");
|
||||
}
|
||||
for table in DIRECT.iter().copied().chain(INDIRECT.iter().map(|v| v.0)).chain(["journal_erased_sessions"]) {
|
||||
let sql: String = c.query_row("SELECT sql FROM sqlite_master WHERE name=?1", [format!("deletion_fence_{table}_UPDATE")], |r| r.get(0))?;
|
||||
assert!(sql.contains("OLD.") && sql.contains("NEW."), "{table}: {sql}");
|
||||
}
|
||||
// OLD fenced -> NEW unfenced must not evade a guard.
|
||||
assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![a, "new-id"]).is_err());
|
||||
assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![b, a]).is_err());
|
||||
assert!(c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&b]).is_err());
|
||||
assert!(c.execute("DELETE FROM subject_deletions WHERE subject_id=?1", [&a]).is_err());
|
||||
assert!(c.execute("UPDATE subject_deletions SET operation_id='changed' WHERE subject_id=?1", [&a]).is_err());
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
store.finish_subject_deletion(&a, "delete").unwrap();
|
||||
assert!(store.staging_candidate(&b, "b").unwrap().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_retired_is_readable_serde_compatible_and_explicitly_deletable() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
let id = store.create_subject(role()).unwrap().id;
|
||||
store.create_memory(&id, draft()).unwrap();
|
||||
legacy_retired_fixture(&store, &id).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_str::<SubjectState>("\"retired\"").unwrap(),
|
||||
SubjectState::LegacyRetired
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&SubjectState::LegacyRetired).unwrap(),
|
||||
"\"retired\""
|
||||
);
|
||||
assert_eq!(store.list_memories(&id).unwrap().len(), 1);
|
||||
assert!(store.create_memory(&id, draft()).is_err());
|
||||
store.begin_subject_deletion(&id, "delete-legacy").unwrap();
|
||||
store.finish_subject_deletion(&id, "delete-legacy").unwrap();
|
||||
assert_eq!(count(&store, "memory_changes"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_preserves_legacy_content_until_explicit_delete_including_archive() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let id: String;
|
||||
{
|
||||
let manager = FeatureStorage::new(temp.path());
|
||||
let scope = manager.workspace("workspace-a").unwrap();
|
||||
let registration = scope
|
||||
.register(FeatureRegistration::new(
|
||||
SUBJEKTIV_FEATURE_ID,
|
||||
&MIGRATIONS[..6],
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
id = "legacy-subject".into();
|
||||
store.database.try_transaction::<_, SubjektivError>(|tx| {
|
||||
let raw = serde_json::json!({"schema_version":1,"id":id,"role":"workspace_companion","behavior_md":"legacy private behavior","state":"active","created_at":"2026-01-01","updated_at":"2026-01-01"}).to_string();
|
||||
tx.execute("INSERT INTO subjects(subject_id,role,state,store_revision,record_json,created_at,updated_at) VALUES(?1,'workspace_companion','active',0,?2,'2026-01-01','2026-01-01')", params![id,raw])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
// Build a saved SQL fixture rather than using current-only APIs against an old schema.
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, SubjektivError>(|tx| {
|
||||
let raw: String = tx.query_row(
|
||||
"SELECT record_json FROM subjects WHERE subject_id=?1",
|
||||
[&id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let mut value: serde_json::Value = serde_json::from_str(&raw)?;
|
||||
value["state"] = "retired".into();
|
||||
tx.execute(
|
||||
"UPDATE subjects SET state='retired',record_json=?2 WHERE subject_id=?1",
|
||||
params![id, value.to_string()],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
}
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
assert_eq!(
|
||||
store.subject(&id).unwrap().unwrap().state,
|
||||
SubjectState::LegacyRetired
|
||||
);
|
||||
assert!(count(&store, "legacy_record_archive") > 1);
|
||||
store.begin_subject_deletion(&id, "delete").unwrap();
|
||||
store.finish_subject_deletion(&id, "delete").unwrap();
|
||||
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||
let remaining: i64 = c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE json_extract(row_json,'$.subject_id')=?1", [&id], |r|r.get(0))?;
|
||||
assert_eq!(remaining, 0);
|
||||
// Workspace-only metadata remains and is not a copy of the Subject's content.
|
||||
assert_eq!(c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE source_table='store_scope'", [], |r|r.get::<_,i64>(0))?, 1);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_deletions_on_independent_connections_save_one_request_identity() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
let (_second_manager, _second_scope, second) = open_store(temp.path(), "workspace-a");
|
||||
let id = store.create_subject(role()).unwrap().id;
|
||||
let barrier = std::sync::Barrier::new(3);
|
||||
std::thread::scope(|scope| {
|
||||
let a = scope.spawn(|| {
|
||||
barrier.wait();
|
||||
store.begin_subject_deletion(&id, "first")
|
||||
});
|
||||
let b = scope.spawn(|| {
|
||||
barrier.wait();
|
||||
second.begin_subject_deletion(&id, "second")
|
||||
});
|
||||
barrier.wait();
|
||||
let results = [a.join().unwrap(), b.join().unwrap()];
|
||||
assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
|
||||
assert_eq!(
|
||||
results
|
||||
.iter()
|
||||
.filter(|r| matches!(r, Err(SubjektivError::SubjectDeletionConflict(_))))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
});
|
||||
let saved = second.subject_deletion(&id).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.begin_subject_deletion(&id, &saved.operation_id)
|
||||
.unwrap(),
|
||||
saved
|
||||
);
|
||||
std::thread::scope(|scope| {
|
||||
let a = scope.spawn(|| store.finish_subject_deletion(&id, &saved.operation_id));
|
||||
let b = scope.spawn(|| second.finish_subject_deletion(&id, &saved.operation_id));
|
||||
assert_eq!(a.join().unwrap().unwrap(), b.join().unwrap().unwrap());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_migration_failure_rolls_back_fences_and_trigger_rewrites_then_retries() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let id;
|
||||
{
|
||||
let manager = FeatureStorage::new(temp.path());
|
||||
let scope = manager.workspace("workspace-a").unwrap();
|
||||
let registration = scope
|
||||
.register(FeatureRegistration::new(
|
||||
SUBJEKTIV_FEATURE_ID,
|
||||
&MIGRATIONS[..13],
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
id = store.create_subject(role()).unwrap().id;
|
||||
store
|
||||
.stage_candidate(candidate(&id, "preserved", store.workspace_id()))
|
||||
.unwrap();
|
||||
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||
// Fail near the end, after retention triggers have already been rewritten.
|
||||
c.execute_batch("CREATE TRIGGER deletion_fence_journal_sources_INSERT BEFORE INSERT ON journal_sources BEGIN SELECT 1; END;")?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
}
|
||||
let manager = FeatureStorage::new(temp.path());
|
||||
let scope = manager.workspace("workspace-a").unwrap();
|
||||
let current = SubjektivStore::register(&scope).unwrap();
|
||||
assert!(SubjektivStore::open(&scope, ¤t).is_err());
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
let manager = FeatureStorage::new(temp.path());
|
||||
let scope = manager.workspace("workspace-a").unwrap();
|
||||
// Reopen only the fixture's pre-deletion registration to inspect the failed migration.
|
||||
let old = scope
|
||||
.register(FeatureRegistration::new(
|
||||
SUBJEKTIV_FEATURE_ID,
|
||||
&MIGRATIONS[..13],
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, &old).unwrap();
|
||||
store
|
||||
.database
|
||||
.try_with_connection::<_, SubjektivError>(|c| {
|
||||
assert!(!c.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions')",
|
||||
[],
|
||||
|r| r.get::<_, bool>(0)
|
||||
)?);
|
||||
let sql: String = c.query_row(
|
||||
"SELECT sql FROM sqlite_master WHERE name='staging_records_no_delete'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
assert!(!sql.contains("subject_deletions"));
|
||||
assert!(
|
||||
c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&id])
|
||||
.is_err()
|
||||
);
|
||||
c.execute_batch("DROP TRIGGER deletion_fence_journal_sources_INSERT;")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
assert!(store.staging_candidate(&id, "preserved").unwrap().is_some());
|
||||
assert!(store.pending_subject_deletions().unwrap().is_empty());
|
||||
store.begin_subject_deletion(&id, "delete").unwrap();
|
||||
store.finish_subject_deletion(&id, "delete").unwrap();
|
||||
}
|
||||
@@ -39,6 +39,7 @@ fn denied() -> JournalError {
|
||||
}
|
||||
|
||||
pub(super) fn check_writer(tx: &Connection, host: &JournalHostContext) -> JournalResult<()> {
|
||||
crate::require_subject_in_connection(tx, &host.subject_id)?;
|
||||
let attr = host.attribution.as_ref().ok_or_else(denied)?;
|
||||
let binding = host.writer_binding_id.as_ref().ok_or_else(denied)?;
|
||||
if !tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings b JOIN journal_session_bindings s ON s.binding_id=b.binding_id WHERE b.binding_id=?1 AND b.subject_id=?2 AND b.runtime_id=?3 AND b.worker_id=?4 AND b.active=1 AND s.session_id=?5)", params![binding,host.subject_id,attr.runtime_id,attr.worker_id,attr.session_id], |r| r.get::<_,bool>(0))? {
|
||||
@@ -63,6 +64,7 @@ impl SubjektivStore {
|
||||
binding: &JournalWriterBinding,
|
||||
) -> JournalResult<Option<String>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||
Ok(tx.query_row("SELECT j.source_session_id FROM journals j JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE j.subject_id=?1 AND s.binding_id=(SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=0 ORDER BY rowid DESC LIMIT 1) ORDER BY j.updated_at DESC,j.id DESC LIMIT 1", [&binding.subject_id], |r| r.get(0)).optional()?)
|
||||
})
|
||||
}
|
||||
@@ -77,6 +79,7 @@ impl SubjektivStore {
|
||||
worker: &str,
|
||||
) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
crate::require_subject_in_connection(tx, subject)?;
|
||||
Ok(tx.query_row(
|
||||
"SELECT NOT EXISTS(SELECT 1 FROM journal_writer_bindings
|
||||
WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)
|
||||
@@ -95,12 +98,14 @@ impl SubjektivStore {
|
||||
worker: &str,
|
||||
) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
crate::require_subject_in_connection(tx, subject)?;
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 AND active=0)", params![subject,runtime,worker], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn journal_writer_active(&self, binding: &JournalWriterBinding) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4 AND active=1)", params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
@@ -111,6 +116,7 @@ impl SubjektivStore {
|
||||
session: &str,
|
||||
) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_session_bindings WHERE binding_id=?1 AND session_id=?2)", params![binding.binding_id,session], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
@@ -135,6 +141,8 @@ impl SubjektivStore {
|
||||
}
|
||||
}
|
||||
self.database.try_transaction(|tx| {
|
||||
crate::require_subject_in_connection(tx, subject)?;
|
||||
crate::deletion::require_live_worker(tx, runtime, worker)?;
|
||||
let state: Option<String> = tx.query_row("SELECT state FROM subjects WHERE subject_id=?1", [subject], |r| r.get(0)).optional()?;
|
||||
if state.as_deref() != Some("active") { return Err(JournalError::PermissionDenied("Subject is absent or retired".into())); }
|
||||
let current: Option<JournalWriterBinding> = tx.query_row("SELECT subject_id,runtime_id,worker_id,binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=1", [subject], |r| Ok(JournalWriterBinding { subject_id:r.get(0)?,runtime_id:r.get(1)?,worker_id:r.get(2)?,binding_id:r.get(3)? })).optional()?;
|
||||
@@ -200,6 +208,7 @@ impl SubjektivStore {
|
||||
/// It never fixes an open entry or declares the represented work complete.
|
||||
pub fn release_journal_writer(&self, binding: &JournalWriterBinding) -> JournalResult<()> {
|
||||
self.database.try_transaction(|tx| {
|
||||
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||
let active: Option<bool> = tx.query_row("SELECT active FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0)).optional()?;
|
||||
let Some(active) = active else { return Err(denied()); };
|
||||
if !active { return Ok(()); }
|
||||
|
||||
@@ -1158,3 +1158,69 @@ fn failed_link_migration_rolls_back_ddl_and_preserves_existing_data_for_retry()
|
||||
assert_eq!(linked(&f.links(&selector, true).sources[0]).1, entry);
|
||||
assert_eq!(f.snapshot(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_denies_all_memory_journal_context_and_reconcile_entry_points() {
|
||||
let f = Fixture::new();
|
||||
let staged = f.stage("candidate", &[0]);
|
||||
let selector = f.apply(&staged);
|
||||
let entry = f.fixed("entry", 0, vec![]);
|
||||
f.links(&selector, true);
|
||||
f.store()
|
||||
.begin_subject_deletion(&f.host.subject_id, "delete")
|
||||
.unwrap();
|
||||
for request in [
|
||||
MemoryJournalRequest::Links {
|
||||
selector: selector.clone(),
|
||||
},
|
||||
MemoryJournalRequest::Reconcile {
|
||||
selector: selector.clone(),
|
||||
},
|
||||
MemoryJournalRequest::LinksPage {
|
||||
selector: selector.clone(),
|
||||
source_offset: 0,
|
||||
limit: 1,
|
||||
},
|
||||
MemoryJournalRequest::ReconcilePage {
|
||||
selector: selector.clone(),
|
||||
source_offset: 0,
|
||||
limit: 1,
|
||||
},
|
||||
MemoryJournalRequest::Read {
|
||||
selector,
|
||||
source_index: 0,
|
||||
journal_id: f.journal_id.clone(),
|
||||
entry_id: entry.clone(),
|
||||
expected_token: None,
|
||||
byte_offset: 0,
|
||||
max_bytes: 8,
|
||||
},
|
||||
MemoryJournalRequest::Related {
|
||||
journal_id: f.journal_id.clone(),
|
||||
entry_id: entry,
|
||||
after: None,
|
||||
limit: 1,
|
||||
},
|
||||
] {
|
||||
assert!(matches!(
|
||||
f.store().execute_memory_journal(&f.host, request),
|
||||
Err(JournalError::Subject(
|
||||
crate::SubjektivError::SubjectNotFound(_)
|
||||
))
|
||||
));
|
||||
}
|
||||
f.store()
|
||||
.finish_subject_deletion(&f.host.subject_id, "delete")
|
||||
.unwrap();
|
||||
f.store()
|
||||
.database
|
||||
.try_with_connection::<_, JournalError>(|c| {
|
||||
assert_eq!(
|
||||
c.query_row("SELECT COUNT(*) FROM memory_journal_links", [], |r| r
|
||||
.get::<_, i64>(0))?,
|
||||
0
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -63,6 +63,10 @@ impl JournalInput {
|
||||
// below operates on this value only, never by making another live capture request.
|
||||
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
|
||||
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
|
||||
// Capture runs outside SQLite; a delayed callback cannot deliver deleted content.
|
||||
store
|
||||
.database
|
||||
.try_with_connection(|tx| super::store::check_host(store, tx, host, true))?;
|
||||
input.selected_ranges = capture_ranges(&input.capture)?;
|
||||
if input.selected_ranges.is_empty() {
|
||||
return Ok(input);
|
||||
|
||||
@@ -469,6 +469,11 @@ fn execute_inner(
|
||||
Ok::<_, JournalError>(())
|
||||
})?;
|
||||
}
|
||||
// Also check empty source reads and Initial snapshot responses (no delivery write).
|
||||
store.database.try_with_connection(|tx| {
|
||||
super::store::job(store, tx, context, false)?;
|
||||
Ok::<_, JournalError>(())
|
||||
})?;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
|
||||
@@ -105,6 +105,7 @@ pub(super) fn check_session_host(
|
||||
"different storage scope".into(),
|
||||
));
|
||||
}
|
||||
crate::require_subject_in_connection(tx, &host.subject_id)?;
|
||||
let attr = host
|
||||
.attribution
|
||||
.as_ref()
|
||||
@@ -134,6 +135,7 @@ pub(super) fn check_host(
|
||||
"different storage scope".into(),
|
||||
));
|
||||
}
|
||||
crate::require_subject_in_connection(tx, &host.subject_id)?;
|
||||
if write {
|
||||
let state: String = tx.query_row(
|
||||
"SELECT state FROM subjects WHERE subject_id=?1",
|
||||
|
||||
@@ -349,7 +349,7 @@ mod input_delivery {
|
||||
open_action("must not save"),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.store().retire_subject(&f.host.subject_id).unwrap();
|
||||
crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap();
|
||||
f.reopen();
|
||||
assert!(matches!(
|
||||
service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges),
|
||||
@@ -1210,6 +1210,21 @@ mod snapshot_inputs {
|
||||
service::finish(f.store(), &recovery).unwrap(),
|
||||
saved_completion
|
||||
);
|
||||
// Deletion, unlike writer release, invalidates even exact saved input/completion replay.
|
||||
f.store()
|
||||
.begin_subject_deletion(&host.subject_id, "delete")
|
||||
.unwrap();
|
||||
subject_deleted(
|
||||
f.store()
|
||||
.prepare_journal_input(&host, &input, generation.job.clone()),
|
||||
);
|
||||
subject_deleted(service::finish(f.store(), &recovery));
|
||||
subject_deleted(JournalJobContext::validated_recovery(
|
||||
f.store(),
|
||||
&host,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2833,7 +2848,7 @@ fn job_service_source_reads_are_generation_bound_and_stop_at_supplied_byte_end()
|
||||
#[test]
|
||||
fn explicit_erasure_remains_available_for_retired_subject_without_reopening_recording() {
|
||||
let f = Fixture::new();
|
||||
f.store().retire_subject(&f.host.subject_id).unwrap();
|
||||
crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap();
|
||||
assert!(f.store().journal_retention_required(&f.host).unwrap());
|
||||
f.store()
|
||||
.erase_journal_session(&f.host, "erase-retired")
|
||||
@@ -4086,10 +4101,7 @@ fn retired_subject_keeps_reader_access_but_rejects_existing_write_context() {
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
fixture
|
||||
.store()
|
||||
.retire_subject(&fixture.host.subject_id)
|
||||
.unwrap();
|
||||
crate::tests::legacy_retired_fixture(fixture.store(), &fixture.host.subject_id).unwrap();
|
||||
assert!(matches!(
|
||||
fixture.store().execute_journal(&ctx, late),
|
||||
Err(JournalError::PermissionDenied(_))
|
||||
@@ -5745,3 +5757,311 @@ fn late_gap_admission_and_partial_updates_reject_stale_source_page_cursors() {
|
||||
Err(JournalError::InvalidInput(_))
|
||||
));
|
||||
}
|
||||
|
||||
fn subject_deleted<T: std::fmt::Debug>(result: JournalResult<T>) {
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(JournalError::Subject(
|
||||
crate::SubjektivError::SubjectNotFound(_)
|
||||
))
|
||||
),
|
||||
"unexpected result: {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_fence_denies_all_journal_read_write_binding_and_recovery_paths() {
|
||||
use transport::{
|
||||
JournalJobRequest as Request, JournalSnapshot, JournalTextRead, JournalTextTarget,
|
||||
};
|
||||
let mut f = Fixture::new();
|
||||
let (generation, ctx) = f.generation("delete-job", f.ranges.clone());
|
||||
let operation = f.operation(
|
||||
"saved",
|
||||
open_action("private body"),
|
||||
vec![f.ranges[0].clone()],
|
||||
);
|
||||
let receipt = f.store().execute_journal(&ctx, operation.clone()).unwrap();
|
||||
let entry = receipt.entry_id.unwrap();
|
||||
let reader = JournalHostContext::validated_reader(f.store(), &f.host.subject_id).unwrap();
|
||||
let writer = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
let digest = format!("sha256:{}", "a".repeat(64));
|
||||
service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges).unwrap();
|
||||
f.store().database.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute("INSERT INTO journal_generation_deliveries(generation_id,source_order,ranges_json) VALUES(?1,0,'[]')", [&generation.id])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
f.store().database.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute("INSERT INTO legacy_record_archive(source_table,row_position,row_json) VALUES('subjects',0,?1)", [serde_json::json!({"subject_id":f.host.subject_id,"record_json":"archived private body"}).to_string()])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
f.store()
|
||||
.begin_subject_deletion(&f.host.subject_id, "delete")
|
||||
.unwrap();
|
||||
// Exercise actual SQL owner resolution for *every* indirect content table,
|
||||
// not only table-name inventory. Fence errors must precede PK/immutable/FK errors.
|
||||
f.store()
|
||||
.database
|
||||
.try_with_connection::<_, JournalError>(|tx| {
|
||||
for (table, field, new_owner) in [
|
||||
("journal_sources", "journal_id", "other-journal"),
|
||||
("journal_operations", "journal_id", "other-journal"),
|
||||
("journal_generations", "journal_id", "other-journal"),
|
||||
("journal_session_bindings", "binding_id", "other-binding"),
|
||||
(
|
||||
"journal_generation_deliveries",
|
||||
"generation_id",
|
||||
"other-generation",
|
||||
),
|
||||
(
|
||||
"journal_generation_input_deliveries",
|
||||
"generation_id",
|
||||
"other-generation",
|
||||
),
|
||||
(
|
||||
"legacy_record_archive",
|
||||
"row_json",
|
||||
"{\"subject_id\":\"other-subject\"}",
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r
|
||||
.get::<_, i64>(0))?
|
||||
> 0,
|
||||
"{table}"
|
||||
);
|
||||
let insert = tx
|
||||
.execute(&format!("INSERT INTO {table} SELECT * FROM {table}"), [])
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
insert.to_string().contains("Subject deletion has begun"),
|
||||
"{table}: {insert}"
|
||||
);
|
||||
let update = tx
|
||||
.execute(&format!("UPDATE {table} SET {field}=?1"), [new_owner])
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
update.to_string().contains("Subject deletion has begun"),
|
||||
"{table}: {update}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
macro_rules! denied { ($($result:expr),+ $(,)?) => { $(subject_deleted($result);)+ }; }
|
||||
for host in [&f.host, &reader] {
|
||||
denied!(
|
||||
f.store().list_journals(host, None, 1),
|
||||
f.store().read_journal(host, &f.journal_id),
|
||||
f.store().list_journal_entries(host, &f.journal_id, None, 1),
|
||||
f.store().read_journal_text(
|
||||
host,
|
||||
&f.journal_id,
|
||||
Some(&entry),
|
||||
JournalTextField::Body,
|
||||
None,
|
||||
0,
|
||||
8
|
||||
),
|
||||
f.store().read_journal_source(host, &f.journal_id, 0),
|
||||
f.store().list_journal_sources(host, &f.journal_id, None, 1),
|
||||
f.store()
|
||||
.query_journal_operation(host, &f.journal_id, "saved"),
|
||||
f.store()
|
||||
.read_journal_operation_status(host, &f.journal_id, "saved"),
|
||||
f.store()
|
||||
.read_journal_operation_coverage(host, &f.journal_id, "saved", 0, 1),
|
||||
f.store()
|
||||
.list_journal_operations(host, &f.journal_id, None, 1),
|
||||
f.store().journal_entry_summary(host, &f.journal_id, &entry),
|
||||
f.store()
|
||||
.journal_correction_targets(host, &f.journal_id, &entry),
|
||||
f.store()
|
||||
.browse_journal_entries(host, None, None, None, None, 1),
|
||||
f.store()
|
||||
.journal_has_unreflected_sources(host, &f.journal_id),
|
||||
f.store().journal_startup_snapshot(host, None),
|
||||
);
|
||||
}
|
||||
denied!(
|
||||
f.store().journal_compaction_snapshot(&f.host),
|
||||
f.store().journal_session_erased(&f.host),
|
||||
f.store().journal_for_session(&f.host),
|
||||
f.store().journal_retention_required(&f.host),
|
||||
f.store().erase_journal_session(&f.host, "erase"),
|
||||
f.store().release_erased_journal_body(&f.host),
|
||||
f.store().journal_predecessor_session(&writer),
|
||||
f.store()
|
||||
.journal_body_is_unpublished(&f.host.subject_id, "runtime", "worker"),
|
||||
f.store()
|
||||
.journal_worker_was_released(&f.host.subject_id, "runtime", "worker"),
|
||||
f.store().journal_writer_active(&writer),
|
||||
f.store()
|
||||
.journal_session_writer_matches(&writer, "source-session"),
|
||||
f.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker"),
|
||||
f.store()
|
||||
.bind_journal_session(&writer, f.host.attribution.clone().unwrap()),
|
||||
f.store().release_journal_writer(&writer),
|
||||
f.store().accept_journal_sources(&f.host, "late", &f.ranges),
|
||||
f.store()
|
||||
.mark_journal_sources_unavailable(&f.host, &f.journal_id, &f.ranges),
|
||||
f.store()
|
||||
.mark_journal_sources_available(&f.host, &f.journal_id, &f.ranges),
|
||||
f.store().active_journal_generation(&f.host, &f.journal_id),
|
||||
f.store().prepare_journal_generation(
|
||||
&f.host,
|
||||
&f.journal_id,
|
||||
generation.job.clone(),
|
||||
f.ranges.clone()
|
||||
),
|
||||
f.store()
|
||||
.close_journal_generation(&f.host, &generation.id, &generation.job),
|
||||
f.store().read_journal_generation(&f.host, &generation.id),
|
||||
f.store().journal_resume_state(&f.host, &f.journal_id),
|
||||
f.store().execute_journal(&ctx, operation.clone()),
|
||||
JournalJobContext::validated_recovery(
|
||||
f.store(),
|
||||
&f.host,
|
||||
&generation.id,
|
||||
generation.job.clone()
|
||||
),
|
||||
service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges),
|
||||
service::finish(f.store(), &ctx),
|
||||
service::execute_snapshot(f.store(), &ctx, &digest, Request::ReadState),
|
||||
service::execute_snapshot(
|
||||
f.store(),
|
||||
&ctx,
|
||||
&digest,
|
||||
Request::QueryOperation {
|
||||
request_id: "saved".into()
|
||||
}
|
||||
),
|
||||
service::execute_snapshot(
|
||||
f.store(),
|
||||
&ctx,
|
||||
&digest,
|
||||
Request::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Position {
|
||||
snapshot: JournalSnapshot::Initial
|
||||
},
|
||||
offset: 0,
|
||||
limit: 8,
|
||||
content_token: None
|
||||
}
|
||||
}
|
||||
),
|
||||
);
|
||||
// Failed purge and restart must not revive any cached context or exact receipt.
|
||||
f.sql("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected'); END;").unwrap();
|
||||
assert!(
|
||||
f.store()
|
||||
.finish_subject_deletion(&f.host.subject_id, "delete")
|
||||
.is_err()
|
||||
);
|
||||
f.reopen();
|
||||
subject_deleted(f.store().execute_journal(&ctx, operation));
|
||||
f.sql("DROP TRIGGER injected_subject_purge_failure;")
|
||||
.unwrap();
|
||||
f.store()
|
||||
.finish_subject_deletion(&f.host.subject_id, "delete")
|
||||
.unwrap();
|
||||
f.store()
|
||||
.database
|
||||
.try_with_connection::<_, JournalError>(|tx| {
|
||||
for table in [
|
||||
"journals",
|
||||
"journal_entries",
|
||||
"journal_corrections",
|
||||
"journal_sources",
|
||||
"journal_generations",
|
||||
"journal_operations",
|
||||
"journal_generation_deliveries",
|
||||
"journal_generation_input_deliveries",
|
||||
"journal_writer_bindings",
|
||||
"journal_session_bindings",
|
||||
] {
|
||||
assert_eq!(
|
||||
tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r
|
||||
.get::<_, i64>(0))?,
|
||||
0,
|
||||
"{table}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_during_capture_rejects_even_empty_delayed_input() {
|
||||
use session_store::capture::CaptureDto;
|
||||
let f = Fixture::new();
|
||||
let result = recording::JournalInput::build(f.store(), &f.host, &f.journal_id, |_, _, _| {
|
||||
f.store()
|
||||
.begin_subject_deletion(&f.host.subject_id, "delete")?;
|
||||
Ok(CaptureDto {
|
||||
version: 1,
|
||||
session_id: "source-session".into(),
|
||||
segment_id: "segment".into(),
|
||||
view_entry_count: 0,
|
||||
entries: vec![],
|
||||
})
|
||||
});
|
||||
subject_deleted(result);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_during_source_callback_cannot_return_or_attest_cached_material() {
|
||||
use transport::{JournalJobRequest as Request, JournalTextRead, JournalTextTarget};
|
||||
let f = Fixture::new();
|
||||
let (_, ctx) = f.generation("delayed-source", f.ranges.clone());
|
||||
let result = service::execute_bounded(
|
||||
f.store(),
|
||||
&ctx,
|
||||
Request::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Source {
|
||||
pointer: f.ranges[0].source.pointer.clone(),
|
||||
},
|
||||
offset: 0,
|
||||
limit: 8,
|
||||
content_token: None,
|
||||
},
|
||||
},
|
||||
|source, start, _| {
|
||||
f.store()
|
||||
.begin_subject_deletion(&f.host.subject_id, "delete")?;
|
||||
Ok(recording::JournalMaterial {
|
||||
range: JournalSourceRange {
|
||||
source: source.clone(),
|
||||
bytes: JournalByteRange {
|
||||
start,
|
||||
end: start + 8,
|
||||
},
|
||||
},
|
||||
text: "private!".into(),
|
||||
})
|
||||
},
|
||||
);
|
||||
subject_deleted(result);
|
||||
f.store()
|
||||
.database
|
||||
.try_with_connection::<_, JournalError>(|tx| {
|
||||
assert_eq!(
|
||||
tx.query_row(
|
||||
"SELECT COUNT(*) FROM journal_generation_deliveries",
|
||||
[],
|
||||
|r| r.get::<_, i64>(0)
|
||||
)?,
|
||||
0
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
+63
-28
@@ -1,7 +1,9 @@
|
||||
#![doc = include_str!("../README.md")]
|
||||
|
||||
pub mod api;
|
||||
pub mod deletion;
|
||||
pub mod job;
|
||||
pub use deletion::{DeletionSubjectJob, SubjectDeletion};
|
||||
pub mod journal;
|
||||
mod legacy_migrations;
|
||||
pub mod permissions;
|
||||
@@ -51,7 +53,9 @@ pub enum SubjektivError {
|
||||
InvalidRecord(String),
|
||||
#[error("subject `{0}` was not found in this Workspace")]
|
||||
SubjectNotFound(String),
|
||||
#[error("subject `{0}` is retired")]
|
||||
#[error("Subject deletion request `{0}` conflicts with a saved deletion")]
|
||||
SubjectDeletionConflict(String),
|
||||
#[error("subject `{0}` is legacy-retired")]
|
||||
SubjectRetired(String),
|
||||
#[error("the selected Worker Profile does not enable subjektiv")]
|
||||
ProfileDisabled,
|
||||
@@ -146,7 +150,10 @@ impl<'de> Deserialize<'de> for SubjectRole {
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SubjectState {
|
||||
Active,
|
||||
Retired,
|
||||
/// Saved pre-deletion lifecycle state. The old `retired` spelling remains readable
|
||||
/// and serializable solely to preserve existing data; no retirement operation exists.
|
||||
#[serde(rename = "retired")]
|
||||
LegacyRetired,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -644,6 +651,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
||||
"bind Journal generation delivery to immutable Job input digest",
|
||||
journal::service::migrate_input_delivery,
|
||||
),
|
||||
FeatureMigration::new(
|
||||
14,
|
||||
"add permanent Subject deletion fences and explicit erasure",
|
||||
deletion::migrate,
|
||||
),
|
||||
];
|
||||
|
||||
pub const REGISTRATION: FeatureRegistration =
|
||||
@@ -772,6 +784,9 @@ impl SubjektivStore {
|
||||
|
||||
pub fn subject(&self, subject_id: &str) -> Result<Option<SubjectRecord>> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
if deletion::is_deleting(connection, subject_id)? {
|
||||
return Ok(None);
|
||||
}
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT record_json FROM subjects WHERE subject_id = ?1",
|
||||
@@ -862,11 +877,13 @@ impl SubjektivStore {
|
||||
self.database.try_with_connection(|connection| {
|
||||
let sql = if after.is_some() {
|
||||
"SELECT record_json FROM subjects
|
||||
WHERE created_at > ?1 OR (created_at = ?1 AND subject_id > ?2)
|
||||
WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id)
|
||||
AND (created_at > ?1 OR (created_at = ?1 AND subject_id > ?2))
|
||||
ORDER BY created_at ASC, subject_id ASC
|
||||
LIMIT ?3"
|
||||
} else {
|
||||
"SELECT record_json FROM subjects
|
||||
WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id)
|
||||
ORDER BY created_at ASC, subject_id ASC
|
||||
LIMIT ?1"
|
||||
};
|
||||
@@ -895,27 +912,6 @@ impl SubjektivStore {
|
||||
})
|
||||
}
|
||||
|
||||
/// Retires a Subject independently of stopping Workers. Memory/history remain unedited, undeleted, and readable.
|
||||
/// Retirement is irreversible; new candidates and Memory changes require an active Subject.
|
||||
pub fn retire_subject(&self, subject_id: &str) -> Result<SubjectRecord> {
|
||||
self.database.try_transaction(|transaction| {
|
||||
let mut subject = require_subject(transaction, subject_id)?;
|
||||
if subject.state == SubjectState::Retired {
|
||||
return Ok(subject);
|
||||
}
|
||||
subject.state = SubjectState::Retired;
|
||||
subject.updated_at = now();
|
||||
let raw = serde_json::to_string(&subject)?;
|
||||
transaction.execute(
|
||||
"UPDATE subjects
|
||||
SET state = 'retired', record_json = ?2, updated_at = ?3
|
||||
WHERE subject_id = ?1",
|
||||
params![subject_id, raw, subject.updated_at],
|
||||
)?;
|
||||
Ok(subject)
|
||||
})
|
||||
}
|
||||
|
||||
/// Saves an immutable candidate without changing committed Memory.
|
||||
/// Replay with the same candidate identity and payload returns the original record; different content conflicts.
|
||||
/// Extraction failure after saving never revokes the candidate. The Host commits the extraction pointer separately.
|
||||
@@ -1003,6 +999,10 @@ impl SubjektivStore {
|
||||
) -> Result<Option<SubjectSessionAttribution>> {
|
||||
validate_label("session id", session_id)?;
|
||||
self.database.try_with_connection(|connection| {
|
||||
if let Some(subject) = deletion::deleted_session_subject(connection, session_id)? {
|
||||
return Err(SubjektivError::SubjectNotFound(subject));
|
||||
}
|
||||
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT record_json FROM subject_session_attributions
|
||||
@@ -1011,7 +1011,12 @@ impl SubjektivStore {
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.optional()?;
|
||||
raw.map(|raw| parse_session_attribution(&raw)).transpose()
|
||||
raw.map(|raw| {
|
||||
let record = parse_session_attribution(&raw)?;
|
||||
deletion::require_not_deleting(connection, &record.subject_id)?;
|
||||
Ok(record)
|
||||
})
|
||||
.transpose()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1021,6 +1026,7 @@ impl SubjektivStore {
|
||||
) -> Result<Vec<SubjectSessionAttribution>> {
|
||||
validate_label("subject id", subject_id)?;
|
||||
self.database.try_with_connection(|connection| {
|
||||
require_subject_in_connection(connection, subject_id)?;
|
||||
let mut statement = connection.prepare(
|
||||
"SELECT record_json FROM subject_session_attributions
|
||||
WHERE subject_id = ?1
|
||||
@@ -1062,6 +1068,7 @@ impl SubjektivStore {
|
||||
}
|
||||
let query_limit = limit.saturating_add(1);
|
||||
self.database.try_with_connection(|connection| {
|
||||
require_subject_in_connection(connection, subject_id)?;
|
||||
let raw = match (session_id, after) {
|
||||
(Some(session_id), _) => {
|
||||
let mut statement = connection.prepare(
|
||||
@@ -1128,6 +1135,7 @@ impl SubjektivStore {
|
||||
candidate_id: &str,
|
||||
) -> Result<Option<SubjectStagingRecord>> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
deletion::require_not_deleting(connection, subject_id)?;
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT record_json FROM staging_records
|
||||
@@ -1292,6 +1300,7 @@ impl SubjektivStore {
|
||||
validate_nonempty("resolution reason", &request.reason)?;
|
||||
let request_raw = serde_json::to_string(&request)?;
|
||||
self.database.try_transaction(|transaction| {
|
||||
deletion::require_not_deleting(transaction, subject_id)?;
|
||||
if let Some((stored_request, stored_receipt)) = transaction
|
||||
.query_row(
|
||||
"SELECT request_json, receipt_json
|
||||
@@ -1426,6 +1435,7 @@ impl SubjektivStore {
|
||||
candidate_id: &str,
|
||||
) -> Result<Option<StagingResolution>> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
deletion::require_not_deleting(connection, subject_id)?;
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT resolution_json FROM staging_resolutions
|
||||
@@ -1440,6 +1450,7 @@ impl SubjektivStore {
|
||||
|
||||
pub fn memory(&self, subject_id: &str, memory_id: &str) -> Result<Option<MemoryRecord>> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
deletion::require_not_deleting(connection, subject_id)?;
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT record_json FROM memory_records
|
||||
@@ -1459,6 +1470,7 @@ impl SubjektivStore {
|
||||
change_id: String,
|
||||
) -> Result<Option<MemoryRecord>> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
deletion::require_not_deleting(connection, subject_id)?;
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT record_json FROM memory_changes
|
||||
@@ -1854,6 +1866,7 @@ impl SubjektivStore {
|
||||
snapshot_id: &str,
|
||||
) -> Result<Option<SurfaceSnapshot>> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
deletion::require_not_deleting(connection, subject_id)?;
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT snapshot_json FROM surface_snapshots
|
||||
@@ -2427,6 +2440,11 @@ fn write_session_attribution(
|
||||
record: SubjectSessionAttribution,
|
||||
raw: String,
|
||||
) -> Result<SubjectSessionAttribution> {
|
||||
deletion::require_live_worker(transaction, &record.runtime_id, &record.worker_id)?;
|
||||
if let Some(subject) = deletion::deleted_session_subject(transaction, &record.session_id)? {
|
||||
return Err(SubjektivError::SubjectNotFound(subject));
|
||||
}
|
||||
|
||||
let existing = transaction
|
||||
.query_row(
|
||||
"SELECT record_json FROM subject_session_attributions WHERE session_id = ?1",
|
||||
@@ -2621,6 +2639,7 @@ fn require_subject_in_connection(
|
||||
connection: &rusqlite::Connection,
|
||||
subject_id: &str,
|
||||
) -> Result<SubjectRecord> {
|
||||
deletion::require_not_deleting(connection, subject_id)?;
|
||||
let raw = connection
|
||||
.query_row(
|
||||
"SELECT record_json FROM subjects WHERE subject_id = ?1",
|
||||
@@ -2637,7 +2656,7 @@ fn require_active_subject(
|
||||
subject_id: &str,
|
||||
) -> Result<SubjectRecord> {
|
||||
let subject = require_subject(transaction, subject_id)?;
|
||||
if subject.state == SubjectState::Retired {
|
||||
if subject.state == SubjectState::LegacyRetired {
|
||||
return Err(SubjektivError::SubjectRetired(subject_id.to_string()));
|
||||
}
|
||||
Ok(subject)
|
||||
@@ -3278,6 +3297,7 @@ fn require_job_generation(
|
||||
job_id: &str,
|
||||
attempt_id: &str,
|
||||
) -> Result<String> {
|
||||
deletion::require_not_deleting(connection, subject_id)?;
|
||||
let change_id: Option<String> = connection.query_row(
|
||||
"SELECT memory_fingerprint FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4",
|
||||
params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0),
|
||||
@@ -3370,6 +3390,21 @@ mod tests {
|
||||
use super::*;
|
||||
use feature_storage::FeatureStorage;
|
||||
|
||||
/// Builds saved pre-deletion lifecycle data; not a current product operation.
|
||||
pub(crate) fn legacy_retired_fixture(
|
||||
store: &SubjektivStore,
|
||||
subject_id: &str,
|
||||
) -> Result<SubjectRecord> {
|
||||
store.database.try_transaction(|tx| {
|
||||
let mut subject = require_subject(tx, subject_id)?;
|
||||
subject.state = SubjectState::LegacyRetired;
|
||||
subject.updated_at = now();
|
||||
tx.execute("UPDATE subjects SET state='retired',record_json=?2,updated_at=?3 WHERE subject_id=?1",
|
||||
params![subject_id,serde_json::to_string(&subject)?,subject.updated_at])?;
|
||||
Ok(subject)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn role() -> SubjectRole {
|
||||
SubjectRole::new("workspace_companion").unwrap()
|
||||
}
|
||||
@@ -5047,8 +5082,8 @@ mod tests {
|
||||
Err(SubjektivError::InvalidStateTransition { .. })
|
||||
));
|
||||
|
||||
let retired = store.retire_subject(&subject.id).unwrap();
|
||||
assert_eq!(retired.state, SubjectState::Retired);
|
||||
let retired = crate::tests::legacy_retired_fixture(&store, &subject.id).unwrap();
|
||||
assert_eq!(retired.state, SubjectState::LegacyRetired);
|
||||
assert!(matches!(
|
||||
store.create_memory(&subject.id, draft("after retirement", "invalid")),
|
||||
Err(SubjektivError::SubjectRetired(_))
|
||||
|
||||
@@ -94,6 +94,10 @@ impl SubjektivStore {
|
||||
pub fn bind_worker_execution_grant(&self, grant: &WorkerExecutionGrant) -> Result<()> {
|
||||
validate_grant(self.scope_id(), grant)?;
|
||||
self.database.try_transaction(|tx| {
|
||||
require_subject(tx, &grant.subject_id)?;
|
||||
for (runtime, worker) in [(&grant.runtime_id, &grant.worker_id), (&grant.origin_runtime_id, &grant.origin_worker_id)] {
|
||||
if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)", params![runtime,worker], |r|r.get::<_,bool>(0))? { return Err(denied("Worker belongs to a deleted Subject")); }
|
||||
}
|
||||
if let Some(existing) = grant_in_connection(tx, &grant.runtime_id, &grant.worker_id)? {
|
||||
return if existing == *grant { Ok(()) } else {
|
||||
Err(SubjektivError::WorkerExecutionGrantConflict(format!("{}/{}", grant.runtime_id, grant.worker_id)))
|
||||
@@ -184,9 +188,12 @@ fn grant_in_connection(
|
||||
runtime: &str,
|
||||
worker: &str,
|
||||
) -> Result<Option<WorkerExecutionGrant>> {
|
||||
// Absence must never become non-Subject fallback authority after purge.
|
||||
deletion::require_live_worker(connection, runtime, worker)?;
|
||||
|
||||
let raw: Option<String> = connection
|
||||
.query_row(
|
||||
"SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2",
|
||||
"SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2 AND NOT EXISTS(SELECT 1 FROM subject_deletions d WHERE d.subject_id=worker_execution_grants.subject_id)",
|
||||
params![runtime, worker],
|
||||
|row| row.get(0),
|
||||
)
|
||||
|
||||
@@ -315,7 +315,7 @@ fn redelegation_requires_current_source_and_explicit_retained_parent_ceiling() {
|
||||
store.worker_execution_grant("runtime", "child").unwrap(),
|
||||
Some(child)
|
||||
);
|
||||
store.retire_subject(&subject).unwrap();
|
||||
crate::tests::legacy_retired_fixture(&store, &subject).unwrap();
|
||||
stale.source_change_id = revoked.change_id;
|
||||
assert!(store.bind_worker_execution_grant(&stale).is_err());
|
||||
}
|
||||
@@ -582,7 +582,7 @@ fn subject_grant_listing_is_scoped_retained_ordered_and_fails_on_overflow() {
|
||||
&update("revoke", Some(&source.change_id), Vec::new()),
|
||||
)
|
||||
.unwrap();
|
||||
store.retire_subject(&subject).unwrap();
|
||||
crate::tests::legacy_retired_fixture(&store, &subject).unwrap();
|
||||
let listed = store
|
||||
.list_subject_worker_execution_grants(&subject)
|
||||
.unwrap();
|
||||
@@ -654,3 +654,51 @@ fn failed_schema_migration_rolls_back_new_tables_and_preserves_old_data() {
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_fences_source_and_child_grant_replays_and_never_falls_back_after_purge() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||
let subject = store.create_subject(role()).unwrap().id;
|
||||
let replacement = update("allow", None, vec![permission("file.read", ".", true)]);
|
||||
let source = store
|
||||
.set_subject_permissions(&subject, &replacement)
|
||||
.unwrap();
|
||||
let child = grant(&subject, &source.change_id, "child");
|
||||
store.bind_worker_execution_grant(&child).unwrap();
|
||||
store.begin_subject_deletion(&subject, "delete").unwrap();
|
||||
assert!(store.subject_permissions(&subject).is_err());
|
||||
assert!(
|
||||
store
|
||||
.set_subject_permissions(&subject, &replacement)
|
||||
.is_err()
|
||||
);
|
||||
assert!(store.bind_worker_execution_grant(&child).is_err());
|
||||
assert!(store.worker_execution_grant("runtime", "child").is_err());
|
||||
assert!(store.worker_subject_deleted("runtime", "body").unwrap());
|
||||
assert!(store.worker_subject_deleted("runtime", "child").unwrap());
|
||||
assert_eq!(store.deletion_subject_workers(&subject).unwrap().len(), 2);
|
||||
store.finish_subject_deletion(&subject, "delete").unwrap();
|
||||
assert!(store.worker_execution_grant("runtime", "child").is_err());
|
||||
assert!(store.worker_execution_grant("runtime", "body").is_err());
|
||||
assert!(
|
||||
store
|
||||
.worker_execution_grant("runtime", "unrelated")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let fresh = store.create_subject(role()).unwrap().id;
|
||||
assert!(store.subject_permissions(&fresh).unwrap().is_none());
|
||||
assert!(
|
||||
store
|
||||
.record_session_attribution(
|
||||
SubjectSessionAttribution::new(&fresh, "runtime", "child", "new-session").unwrap()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.claim_journal_writer(&fresh, "runtime", "body")
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -727,6 +727,23 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||
Err("execution backend cannot confirm Subject create execution absence".into())
|
||||
}
|
||||
|
||||
/// Read-only exact execution identity check before Runtime persists exclusion.
|
||||
fn preflight_subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
Err("execution backend cannot validate Subject Worker identity".into())
|
||||
}
|
||||
|
||||
/// Runtime has durably fenced admission. Confirm shutdown and discard all
|
||||
/// execution candidates under the backend Worker lock; uncertainty must fail.
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
Err("execution backend cannot confirm Subject Worker shutdown".into())
|
||||
}
|
||||
|
||||
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
|
||||
fn subject_release(
|
||||
&self,
|
||||
@@ -1069,6 +1086,20 @@ impl WorkerExecutionBackendRef {
|
||||
self.backend.recover_subject_create(request)
|
||||
}
|
||||
|
||||
pub(crate) fn preflight_subject_delete_worker(
|
||||
&self,
|
||||
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
self.backend.preflight_subject_delete_worker(request)
|
||||
}
|
||||
|
||||
pub(crate) fn subject_delete_worker(
|
||||
&self,
|
||||
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
self.backend.subject_delete_worker(request)
|
||||
}
|
||||
|
||||
pub(crate) fn subject_release(
|
||||
&self,
|
||||
request: &runtime_api::SubjectReleaseRequest,
|
||||
|
||||
@@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
|
||||
#[path = "test_support.rs"]
|
||||
pub mod test_support;
|
||||
|
||||
pub(crate) const SCHEMA_VERSION: u32 = 14;
|
||||
pub(crate) const SCHEMA_VERSION: u32 = 15;
|
||||
const RESTORE_REPAIR_SCHEMA_VERSION: u32 = 14;
|
||||
const RESTORE_DECISION_SCHEMA_VERSION: u32 = 13;
|
||||
const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12;
|
||||
// Frozen source schemas: do not move their meaning when adding a migration.
|
||||
@@ -560,6 +561,22 @@ impl FsRuntimeStore {
|
||||
self.root.join(RUNTIME_FILE)
|
||||
}
|
||||
|
||||
pub(crate) fn worker_originals_absent(
|
||||
&self,
|
||||
worker_id: WorkerId,
|
||||
) -> Result<bool, RuntimeError> {
|
||||
let path = self.worker_dir(&worker_id);
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(_) => Ok(false),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true),
|
||||
Err(source) => Err(RuntimeError::StoreIo {
|
||||
operation: "confirm Subject Worker original absence",
|
||||
path,
|
||||
source,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf {
|
||||
self.root.join(WORKERS_DIR).join(worker_id.to_string())
|
||||
}
|
||||
@@ -659,6 +676,8 @@ pub(crate) struct OpenedFsRuntimeStore {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct PersistedRuntimeState {
|
||||
pub(crate) subject_delete_worker_fences:
|
||||
BTreeMap<WorkerId, crate::runtime::SubjectDeleteWorkerFence>,
|
||||
pub(crate) subject_create_recovery_fences:
|
||||
BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
|
||||
pub(crate) subject_releases: Vec<(
|
||||
@@ -876,11 +895,12 @@ fn plan_runtime_store_migration(
|
||||
| SOURCE_SCHEMA_VERSION
|
||||
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||
| RESTORE_DECISION_SCHEMA_VERSION
|
||||
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||
) {
|
||||
return Err(runtime_store_corrupt(
|
||||
&runtime_path,
|
||||
format!(
|
||||
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
|
||||
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, {RESTORE_REPAIR_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
|
||||
),
|
||||
));
|
||||
}
|
||||
@@ -1216,6 +1236,7 @@ fn migrate_worker_record(
|
||||
| SOURCE_SCHEMA_VERSION
|
||||
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||
| RESTORE_DECISION_SCHEMA_VERSION
|
||||
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||
) {
|
||||
// Admission identity survives migration. Older pending records did not
|
||||
// distinguish a committed decision from preparation or failed cleanup.
|
||||
@@ -1229,10 +1250,12 @@ fn migrate_worker_record(
|
||||
"Worker schema does not match Runtime migration source".into(),
|
||||
));
|
||||
}
|
||||
if source_schema_version != RESTORE_DECISION_SCHEMA_VERSION {
|
||||
if source_schema_version < RESTORE_DECISION_SCHEMA_VERSION {
|
||||
migrate_restore_commit_fact(&mut identity_document, identity_path)?;
|
||||
}
|
||||
if source_schema_version < RESTORE_REPAIR_SCHEMA_VERSION {
|
||||
migrate_restore_repair_fact(&mut identity_document, identity_path)?;
|
||||
}
|
||||
identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION);
|
||||
let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document)
|
||||
.map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?;
|
||||
@@ -1691,6 +1714,9 @@ fn migrate_runtime_document(
|
||||
"schema_version".to_string(),
|
||||
serde_json::Value::from(SCHEMA_VERSION),
|
||||
);
|
||||
object
|
||||
.entry("subject_delete_worker_fences")
|
||||
.or_insert_with(|| serde_json::json!({}));
|
||||
// Intermediate schema-11 writers may already have saved irreversible fences.
|
||||
// Default only a missing field; malformed or existing fences must not be erased.
|
||||
object
|
||||
@@ -1698,7 +1724,10 @@ fn migrate_runtime_document(
|
||||
.or_insert_with(|| serde_json::json!({}));
|
||||
if !matches!(
|
||||
source_schema_version,
|
||||
SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION
|
||||
SOURCE_SCHEMA_VERSION
|
||||
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||
| RESTORE_DECISION_SCHEMA_VERSION
|
||||
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||
) {
|
||||
object.remove("workers");
|
||||
// This legacy cache is not Runtime lifecycle or restore authority. Bundles
|
||||
@@ -2249,6 +2278,7 @@ fn migrate_runtime_store_in_place(
|
||||
SOURCE_SCHEMA_VERSION
|
||||
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||
| RESTORE_DECISION_SCHEMA_VERSION
|
||||
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||
)
|
||||
{
|
||||
atomic_write_json(
|
||||
@@ -2288,6 +2318,7 @@ fn migrate_runtime_store_in_place(
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
struct RuntimeSnapshot {
|
||||
subject_delete_worker_fences: BTreeMap<WorkerId, crate::runtime::SubjectDeleteWorkerFence>,
|
||||
#[serde(default)]
|
||||
subject_create_recovery_fences: BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
|
||||
#[serde(default)]
|
||||
@@ -2326,6 +2357,7 @@ fn record_worker_load_diagnostic(
|
||||
impl RuntimeSnapshot {
|
||||
fn from_persisted(state: &PersistedRuntimeState) -> Self {
|
||||
Self {
|
||||
subject_delete_worker_fences: state.subject_delete_worker_fences.clone(),
|
||||
subject_create_recovery_fences: state.subject_create_recovery_fences.clone(),
|
||||
subject_releases: state.subject_releases.clone(),
|
||||
schema_version: SCHEMA_VERSION,
|
||||
@@ -2350,6 +2382,33 @@ impl RuntimeSnapshot {
|
||||
),
|
||||
});
|
||||
}
|
||||
for (worker_id, fence) in &self.subject_delete_worker_fences {
|
||||
let request = &fence.request;
|
||||
let response_matches = fence.response.as_ref().is_none_or(|response| {
|
||||
response.worker_id == request.worker_id
|
||||
&& response.workspace_id == request.workspace_id
|
||||
&& response.create_fingerprint == request.create_fingerprint
|
||||
&& response.operation_id == request.operation_id
|
||||
});
|
||||
if *worker_id != request.worker_id
|
||||
|| request.workspace_id.trim().is_empty()
|
||||
|| request.create_fingerprint.trim().is_empty()
|
||||
|| request.create_fingerprint.len() > 512
|
||||
|| request.operation_id.trim().is_empty()
|
||||
|| request.operation_id.len() > 512
|
||||
|| self
|
||||
.workspace_owners
|
||||
.get(&request.workspace_id)
|
||||
.is_none_or(|owner| owner.trim().is_empty())
|
||||
|| !response_matches
|
||||
{
|
||||
return Err(RuntimeError::StoreCorrupt {
|
||||
operation: "read Subject delete Worker fence",
|
||||
path: path.to_path_buf(),
|
||||
message: "invalid deletion identity, receipt or Workspace owner".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
for (worker_id, fence) in &self.subject_create_recovery_fences {
|
||||
let request = &fence.request;
|
||||
let response_matches = fence.response.as_ref().is_none_or(|response| {
|
||||
@@ -2392,6 +2451,7 @@ impl RuntimeSnapshot {
|
||||
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
|
||||
) -> PersistedRuntimeState {
|
||||
PersistedRuntimeState {
|
||||
subject_delete_worker_fences: self.subject_delete_worker_fences,
|
||||
subject_create_recovery_fences: self.subject_create_recovery_fences,
|
||||
subject_releases: self.subject_releases,
|
||||
display_name: self.display_name,
|
||||
@@ -3218,8 +3278,7 @@ mod tests {
|
||||
assert_eq!(fs::read(&worker_path).unwrap(), worker_before);
|
||||
// The direct reader still rejects a previous schema: migration is
|
||||
// explicit, not permissive deserialization of a newer format.
|
||||
let snapshot: RuntimeSnapshot = serde_json::from_value(runtime.clone()).unwrap();
|
||||
assert!(snapshot.validate(&runtime_path).is_err());
|
||||
assert!(serde_json::from_value::<RuntimeSnapshot>(runtime.clone()).is_err());
|
||||
|
||||
let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap();
|
||||
let state = opened.state.as_ref().unwrap();
|
||||
@@ -3231,6 +3290,7 @@ mod tests {
|
||||
expected_worker
|
||||
);
|
||||
runtime["schema_version"] = serde_json::json!(SCHEMA_VERSION);
|
||||
runtime["subject_delete_worker_fences"] = serde_json::json!({});
|
||||
if fence_state == "absent" {
|
||||
runtime["subject_create_recovery_fences"] = serde_json::json!({});
|
||||
}
|
||||
@@ -4072,6 +4132,8 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
include!("subject_delete_store_tests.rs");
|
||||
|
||||
fn canonical_worker_store() -> (
|
||||
tempfile::TempDir,
|
||||
FsRuntimeStore,
|
||||
|
||||
@@ -2102,7 +2102,11 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
||||
{
|
||||
return Some("workers:input");
|
||||
}
|
||||
if path == "/v1/subject-create-recovery" && *method == Method::POST {
|
||||
if matches!(
|
||||
path,
|
||||
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
|
||||
) && *method == Method::POST
|
||||
{
|
||||
return Some("workers:delete");
|
||||
}
|
||||
if path.ends_with("/stop")
|
||||
@@ -3332,6 +3336,18 @@ mod tests {
|
||||
struct AcceptingBackend;
|
||||
|
||||
impl WorkerExecutionBackend for AcceptingBackend {
|
||||
fn preflight_subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectCreateRecoveryRequest,
|
||||
|
||||
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
}
|
||||
}
|
||||
|
||||
async fn subject_delete_worker(
|
||||
&self,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, runtime_api::RuntimeApiError> {
|
||||
let scope = material_scope(&self.state, &request.workspace_id)?;
|
||||
self.state
|
||||
.runtime
|
||||
.subject_delete_worker_scoped(&scope, request)
|
||||
.map_err(RuntimeHttpRestError::runtime)
|
||||
.map_err(api_error)
|
||||
}
|
||||
|
||||
async fn recover_subject_create(
|
||||
&self,
|
||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||
|
||||
@@ -1222,6 +1222,54 @@ async fn subject_create_recovery_signed_http_requires_delete_and_matching_worksp
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_delete_worker_signed_http_requires_delete_and_matching_workspace() {
|
||||
let f = Fixture::new();
|
||||
let identity = RuntimeIdentityMaterial::generate("recovery-key").unwrap();
|
||||
let trust = workspace_trust(&identity);
|
||||
let app = runtime_http_router_with_workspace_auth(
|
||||
f.runtime.clone(),
|
||||
None,
|
||||
workspace_test_auth(vec![trust.clone()]),
|
||||
);
|
||||
let path = "/v1/subject-delete-worker";
|
||||
let mut request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: f.scope.workspace_id.clone(),
|
||||
worker_id: WorkerId::now_v7(),
|
||||
create_fingerprint: "unpublished-create".into(),
|
||||
operation_id: "recovery-1".into(),
|
||||
};
|
||||
for operation in ["workers:read", "workers:stop", "workers:create"] {
|
||||
let body = serde_json::to_vec(&request).unwrap();
|
||||
let mut claims = signed_claims(&trust, "POST", path, &body);
|
||||
claims.operation = operation.into();
|
||||
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
request.workspace_id = "other-workspace".into();
|
||||
let body = serde_json::to_vec(&request).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
request.workspace_id = f.scope.workspace_id.clone();
|
||||
for _ in 0..2 {
|
||||
let body = serde_json::to_vec(&request).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
assert_eq!(claims.operation, "workers:delete");
|
||||
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let bytes = axum::body::to_bytes(response.into_body(), 16384)
|
||||
.await
|
||||
.unwrap();
|
||||
let receipt: runtime_api::SubjectDeleteWorkerResponse =
|
||||
serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(receipt.worker_id, request.worker_id);
|
||||
assert_eq!(receipt.workspace_id, request.workspace_id);
|
||||
assert_eq!(receipt.create_fingerprint, request.create_fingerprint);
|
||||
assert_eq!(receipt.operation_id, request.operation_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() {
|
||||
let f = Fixture::new();
|
||||
|
||||
@@ -97,7 +97,7 @@ fn runtime_fixture(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId)
|
||||
"schema_version": schema, "display_name": "retained Runtime",
|
||||
"backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
|
||||
"config_bundles": {}, "workspace_owners": {"workspace-a": "owner-a"},
|
||||
"diagnostics": [], "subject_releases": [],
|
||||
"diagnostics": [], "subject_releases": [], "subject_delete_worker_fences": {},
|
||||
"subject_create_recovery_fences": {fenced_worker.to_string(): {
|
||||
"request": {"workspace_id": "workspace-a", "worker_id": fenced_worker,
|
||||
"create_fingerprint": "retained-create", "operation_id": "retained-recovery"},
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
mod subject_delete;
|
||||
pub(crate) use subject_delete::SubjectDeleteWorkerFence;
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
mod restore_checkpoint;
|
||||
mod restore_continuation;
|
||||
@@ -5645,6 +5647,7 @@ struct RuntimeState {
|
||||
// Ephemeral write barrier, not authority: only canonical reread + sync can
|
||||
// release it. Prevents another mutation from persisting an older projection.
|
||||
unreconciled_worker_writes: std::collections::BTreeSet<WorkerId>,
|
||||
subject_delete_worker_fences: BTreeMap<WorkerId, SubjectDeleteWorkerFence>,
|
||||
subject_create_recovery_fences: BTreeMap<WorkerId, SubjectCreateRecoveryFence>,
|
||||
subject_release_operations: BTreeMap<
|
||||
(WorkerId, String),
|
||||
@@ -5686,6 +5689,7 @@ impl RuntimeState {
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
unreconciled_worker_writes: Default::default(),
|
||||
subject_delete_worker_fences: BTreeMap::new(),
|
||||
subject_create_recovery_fences: BTreeMap::new(),
|
||||
subject_release_operations: BTreeMap::new(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
@@ -5722,6 +5726,7 @@ impl RuntimeState {
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
unreconciled_worker_writes: Default::default(),
|
||||
subject_delete_worker_fences: BTreeMap::new(),
|
||||
subject_create_recovery_fences: BTreeMap::new(),
|
||||
subject_release_operations: BTreeMap::new(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
@@ -5773,6 +5778,7 @@ impl RuntimeState {
|
||||
next_diagnostic_id,
|
||||
workers,
|
||||
unreconciled_worker_writes: Default::default(),
|
||||
subject_delete_worker_fences: persisted.subject_delete_worker_fences,
|
||||
subject_create_recovery_fences: persisted.subject_create_recovery_fences,
|
||||
subject_release_operations: persisted
|
||||
.subject_releases
|
||||
@@ -5804,6 +5810,7 @@ impl RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
fn persisted_state(&self) -> PersistedRuntimeState {
|
||||
PersistedRuntimeState {
|
||||
subject_delete_worker_fences: self.subject_delete_worker_fences.clone(),
|
||||
subject_create_recovery_fences: self.subject_create_recovery_fences.clone(),
|
||||
subject_releases: self.subject_release_operations.values().cloned().collect(),
|
||||
display_name: self.display_name.clone(),
|
||||
@@ -6027,7 +6034,9 @@ impl RuntimeState {
|
||||
}
|
||||
|
||||
fn ensure_subject_create_not_fenced(&self, worker_id: WorkerId) -> Result<(), RuntimeError> {
|
||||
if self.subject_create_recovery_fences.contains_key(&worker_id) {
|
||||
if self.subject_create_recovery_fences.contains_key(&worker_id)
|
||||
|| self.subject_delete_worker_fences.contains_key(&worker_id)
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject create identity is irreversibly fenced".into(),
|
||||
));
|
||||
@@ -6040,6 +6049,14 @@ impl RuntimeState {
|
||||
worker: &WorkerRef,
|
||||
allow_disconnected: bool,
|
||||
) -> Result<(), RuntimeError> {
|
||||
if allow_disconnected
|
||||
&& self
|
||||
.subject_delete_worker_fences
|
||||
.get(&worker.worker_id)
|
||||
.is_some_and(|fence| fence.response.is_some())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
self.ensure_subject_create_not_fenced(worker.worker_id)?;
|
||||
use runtime_api::SubjectReleaseState as State;
|
||||
if self
|
||||
@@ -6110,6 +6127,13 @@ impl RuntimeState {
|
||||
}
|
||||
|
||||
fn forget_workspace_owner_if_unused(&mut self, workspace_id: &str) -> bool {
|
||||
if self
|
||||
.subject_delete_worker_fences
|
||||
.values()
|
||||
.any(|fence| fence.request.workspace_id == workspace_id)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.subject_create_recovery_fences
|
||||
.values()
|
||||
@@ -8767,6 +8791,8 @@ mod tests {
|
||||
|
||||
#[derive(Default)]
|
||||
struct TestExecutionBackend {
|
||||
deletion_confirmed: AtomicBool,
|
||||
deletion_checks: std::sync::atomic::AtomicUsize,
|
||||
recovery_absent: AtomicBool,
|
||||
recovery_checks: std::sync::atomic::AtomicUsize,
|
||||
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
||||
@@ -8848,6 +8874,23 @@ mod tests {
|
||||
}
|
||||
|
||||
impl WorkerExecutionBackend for TestExecutionBackend {
|
||||
fn preflight_subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
self.deletion_checks.fetch_add(1, Ordering::SeqCst);
|
||||
if self.deletion_confirmed.load(Ordering::SeqCst) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("shutdown is unconfirmed".into())
|
||||
}
|
||||
}
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -14074,7 +14117,7 @@ mod tests {
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains(
|
||||
"unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, or 14"
|
||||
"unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, 14, or 15"
|
||||
));
|
||||
|
||||
let _ = std::fs::remove_dir_all(root);
|
||||
@@ -14259,6 +14302,8 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
include!("runtime/subject_delete_tests.rs");
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn subject_create_recovery_is_durable_exact_and_rechecks_absence() {
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
use super::*;
|
||||
|
||||
/// Permanent exclusion survives WorkerRemoval. A receipt never substitutes for a
|
||||
/// fresh execution reconciliation on exact retry, including after Runtime reopen.
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct SubjectDeleteWorkerFence {
|
||||
pub(crate) request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
pub(crate) response: Option<runtime_api::SubjectDeleteWorkerResponse>,
|
||||
}
|
||||
|
||||
impl Runtime {
|
||||
pub fn subject_delete_worker_scoped(
|
||||
&self,
|
||||
scope: &RuntimeWorkspaceScope,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeError> {
|
||||
if request.workspace_id != scope.workspace_id
|
||||
|| request.create_fingerprint.trim().is_empty()
|
||||
|| request.operation_id.trim().is_empty()
|
||||
|| request.create_fingerprint.len() > 512
|
||||
|| request.operation_id.len() > 512
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"invalid Subject delete Worker identity".into(),
|
||||
));
|
||||
}
|
||||
let lease = self.worker_operation_lock(request.worker_id)?;
|
||||
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
let worker_ref = WorkerRef::new(request.worker_id);
|
||||
let backend = {
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
if state.backend == RuntimeBackendKind::Memory {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject delete requires durable storage".into(),
|
||||
));
|
||||
}
|
||||
state.ensure_workspace_owner(scope, false)?;
|
||||
if state.workers.contains_key(&request.worker_id) {
|
||||
state.refresh_uncertain_worker_write(&worker_ref)?;
|
||||
let worker = state.worker(&worker_ref)?;
|
||||
if worker.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|
||||
|| worker.request.as_ref().is_none_or(|create| {
|
||||
create.create_fingerprint != request.create_fingerprint
|
||||
})
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject delete Worker identity mismatch".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(recovery) = state.subject_create_recovery_fences.get(&request.worker_id)
|
||||
&& (recovery.request.workspace_id != request.workspace_id
|
||||
|| recovery.request.create_fingerprint != request.create_fingerprint)
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject recovery identity mismatch".into(),
|
||||
));
|
||||
}
|
||||
if state.subject_delete_worker_fences.values().any(|fence| {
|
||||
fence.request.workspace_id == request.workspace_id
|
||||
&& fence.request.operation_id == request.operation_id
|
||||
&& fence.request.worker_id != request.worker_id
|
||||
}) {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject delete operation reused for another Worker".into(),
|
||||
));
|
||||
}
|
||||
state.execution_backend.clone()
|
||||
};
|
||||
// Unpublished execution identity is checked without holding the catalog
|
||||
// mutex. The Worker operation lock excludes Runtime create/restore.
|
||||
if let Some(backend) = backend.as_ref() {
|
||||
backend
|
||||
.preflight_subject_delete_worker(&request)
|
||||
.map_err(RuntimeError::InvalidRequest)?;
|
||||
}
|
||||
{
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_workspace_owner(scope, false)?;
|
||||
if state.subject_delete_worker_fences.values().any(|fence| {
|
||||
fence.request.workspace_id == request.workspace_id
|
||||
&& fence.request.operation_id == request.operation_id
|
||||
&& fence.request.worker_id != request.worker_id
|
||||
}) {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject delete operation reused for another Worker".into(),
|
||||
));
|
||||
}
|
||||
if let Some(saved) = state.subject_delete_worker_fences.get(&request.worker_id) {
|
||||
if saved.request != request {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"different Subject delete Worker request retained".into(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
state.ensure_workspace_owner(scope, true)?;
|
||||
state.subject_delete_worker_fences.insert(
|
||||
request.worker_id,
|
||||
SubjectDeleteWorkerFence {
|
||||
request: request.clone(),
|
||||
response: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
// Keep exclusion on persistence uncertainty; never start execution
|
||||
// cleanup until this durable write has been confirmed.
|
||||
state.persist_runtime_snapshot()?;
|
||||
}
|
||||
let backend = backend.ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest(
|
||||
"Subject Worker shutdown cannot be confirmed without execution backend".into(),
|
||||
)
|
||||
})?;
|
||||
backend
|
||||
.subject_delete_worker(&request)
|
||||
.map_err(RuntimeError::InvalidRequest)?;
|
||||
let mut state = self.lock()?;
|
||||
if let Some(mut worker) = state.workers.get(&request.worker_id).cloned() {
|
||||
let restore_receipt =
|
||||
RuntimeWorkerRestoreResult::superseded_by_stop(worker.pending_restore);
|
||||
worker.pending_stop = None;
|
||||
worker.pending_restore = None;
|
||||
if let Some(context) = worker.restore_candidate_context.take() {
|
||||
context.discard_candidate();
|
||||
}
|
||||
worker.restore_guard.rotate();
|
||||
worker.restore_guard.finish(restore_receipt);
|
||||
worker.status = WorkerStatus::Stopped;
|
||||
worker.worker_state = None;
|
||||
worker.restore_intent = WorkerRestoreIntent::Explicit;
|
||||
worker.internal_workers.clear();
|
||||
worker.execution_bound = false;
|
||||
state.persist_worker_record(&worker)?;
|
||||
state.workers.insert(request.worker_id, worker);
|
||||
state.publish_worker_upsert(request.worker_id)?;
|
||||
}
|
||||
// This is storage-owner evidence under the same lock that permanently
|
||||
// excludes create/restore, not an inference from catalog projection.
|
||||
#[cfg(feature = "fs-store")]
|
||||
let originals_absent = !state.workers.contains_key(&request.worker_id)
|
||||
&& state
|
||||
.fs_store()
|
||||
.ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest(
|
||||
"Subject original absence requires fs storage".into(),
|
||||
)
|
||||
})?
|
||||
.worker_originals_absent(request.worker_id)?;
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
let originals_absent = false;
|
||||
let response = runtime_api::SubjectDeleteWorkerResponse {
|
||||
workspace_id: request.workspace_id.clone(),
|
||||
worker_id: request.worker_id,
|
||||
create_fingerprint: request.create_fingerprint.clone(),
|
||||
operation_id: request.operation_id.clone(),
|
||||
originals_absent,
|
||||
};
|
||||
state
|
||||
.subject_delete_worker_fences
|
||||
.get_mut(&request.worker_id)
|
||||
.expect("delete fence retained under Worker operation lock")
|
||||
.response = Some(response.clone());
|
||||
state.persist_runtime_snapshot()?;
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
// Runtime component tests use real filesystem persistence and replace only the
|
||||
// external execution boundary. Actual Controller shutdown is tested by the adapter.
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn subject_delete_reserved_id_survives_unknown_reopen_and_rejects_identity_changes() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let options =
|
||||
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
|
||||
let scoped = scope("workspace-a", "server-a");
|
||||
let create = scoped_task_request("delayed create", "workspace-a");
|
||||
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: scoped.workspace_id.clone(),
|
||||
worker_id: create.worker_id,
|
||||
create_fingerprint: create.create_fingerprint.clone(),
|
||||
operation_id: "subject-delete-1".into(),
|
||||
};
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let runtime =
|
||||
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id]
|
||||
.response
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.create_worker_scoped(&scoped, create.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(backend.deletion_checks.load(Ordering::SeqCst), 1);
|
||||
drop(runtime);
|
||||
let runtime =
|
||||
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.create_worker_scoped(&scoped, create.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||
let receipt = runtime
|
||||
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||
.unwrap();
|
||||
let mut expected = serde_json::to_value(&request).unwrap();
|
||||
expected["originals_absent"] = true.into();
|
||||
assert_eq!(serde_json::to_value(&receipt).unwrap(), expected);
|
||||
for index in 0..4 {
|
||||
let mut changed = request.clone();
|
||||
match index {
|
||||
0 => changed.worker_id = WorkerId::now_v7(),
|
||||
1 => changed.operation_id.push_str("-changed"),
|
||||
2 => changed.create_fingerprint.push_str("-changed"),
|
||||
_ => changed.workspace_id.push_str("-changed"),
|
||||
}
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, changed)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scope("workspace-a", "server-b"), request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scope("workspace-b", "server-a"), request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
!runtime
|
||||
.lock()
|
||||
.unwrap()
|
||||
.forget_workspace_owner_if_unused("workspace-a")
|
||||
);
|
||||
drop(runtime);
|
||||
// Even a saved receipt must not manufacture fresh shutdown proof on reopen.
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
|
||||
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request)
|
||||
.unwrap(),
|
||||
receipt
|
||||
);
|
||||
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn subject_delete_existing_stopped_or_active_worker_preserves_record_and_allows_canonical_removal()
|
||||
{
|
||||
for stopped in [false, true] {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
|
||||
.with_runtime_id("test-runtime");
|
||||
let runtime =
|
||||
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let scoped = scope("workspace-a", "server-a");
|
||||
let create = scoped_task_request("subject body", "workspace-a");
|
||||
let worker = runtime
|
||||
.create_worker_scoped(&scoped, create.clone())
|
||||
.unwrap();
|
||||
if stopped {
|
||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||
}
|
||||
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: scoped.workspace_id.clone(),
|
||||
worker_id: create.worker_id,
|
||||
create_fingerprint: create.create_fingerprint.clone(),
|
||||
operation_id: "delete-body".into(),
|
||||
};
|
||||
let mut mismatch = request.clone();
|
||||
mismatch.create_fingerprint.push_str("-wrong");
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, mismatch)
|
||||
.is_err()
|
||||
);
|
||||
let mut mismatch = request.clone();
|
||||
mismatch.workspace_id = "workspace-b".into();
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scope("workspace-b", "server-b"), mismatch)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
!runtime
|
||||
.lock()
|
||||
.unwrap()
|
||||
.subject_delete_worker_fences
|
||||
.contains_key(&create.worker_id)
|
||||
);
|
||||
// A lost Worker settlement cannot be reported as completed deletion.
|
||||
runtime
|
||||
.test_runtime_store()
|
||||
.fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename);
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
let receipt = runtime
|
||||
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||
.unwrap();
|
||||
assert!(!receipt.originals_absent, "a stopped catalog record still has original material");
|
||||
assert_eq!(
|
||||
runtime.worker_detail(&worker.worker_ref).unwrap().status,
|
||||
WorkerStatus::Stopped
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&worker.worker_ref,
|
||||
runtime.test_restore_request(&worker.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.create_worker_scoped(&scoped, create.clone())
|
||||
.is_err()
|
||||
);
|
||||
let record = runtime
|
||||
.test_runtime_store()
|
||||
.read_worker_record(&worker.worker_ref)
|
||||
.unwrap();
|
||||
assert_eq!(record.worker_id, worker.worker_ref.worker_id);
|
||||
drop(runtime);
|
||||
let runtime =
|
||||
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&worker.worker_ref,
|
||||
runtime.test_restore_request(&worker.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request)
|
||||
.unwrap(),
|
||||
receipt
|
||||
);
|
||||
runtime
|
||||
.delete_worker_scoped(&scoped, &worker.worker_ref)
|
||||
.unwrap();
|
||||
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||
assert!(
|
||||
!runtime
|
||||
.lock()
|
||||
.unwrap()
|
||||
.forget_workspace_owner_if_unused("workspace-a")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn subject_delete_unavailable_backend_never_issues_receipt_but_preserves_exclusion() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let options =
|
||||
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
|
||||
let scoped = scope("workspace-a", "server-a");
|
||||
let create = scoped_task_request("reserved without backend", "workspace-a");
|
||||
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: scoped.workspace_id.clone(),
|
||||
worker_id: create.worker_id,
|
||||
create_fingerprint: create.create_fingerprint.clone(),
|
||||
operation_id: "delete-offline".into(),
|
||||
};
|
||||
let runtime = Runtime::with_fs_store(options.clone()).unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id]
|
||||
.response
|
||||
.is_none()
|
||||
);
|
||||
drop(runtime);
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap();
|
||||
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scoped, request)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn subject_delete_missing_catalog_does_not_claim_residual_originals_absent() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap();
|
||||
let scoped = scope("workspace-a", "server-a");
|
||||
let create = scoped_task_request("unpublished original", "workspace-a");
|
||||
let original = root.path().join("workers").join(create.worker_id.to_string()).join("session/original.jsonl");
|
||||
std::fs::create_dir_all(original.parent().unwrap()).unwrap();
|
||||
std::fs::write(&original, "preserve original text").unwrap();
|
||||
let receipt = runtime.subject_delete_worker_scoped(&scoped, runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: scoped.workspace_id.clone(), worker_id: create.worker_id,
|
||||
create_fingerprint: create.create_fingerprint.clone(), operation_id: "delete-unpublished".into(),
|
||||
}).unwrap();
|
||||
assert!(!receipt.originals_absent);
|
||||
assert_eq!(std::fs::read_to_string(original).unwrap(), "preserve original text");
|
||||
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
// Integration of Runtime's durable fence with the real adapter and Controller;
|
||||
// MockFactory replaces only model/provider/profile construction for a hanging run.
|
||||
#[test]
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn subject_delete_running_controller_confirms_shutdown_and_preserves_history() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let client = MockClient::sequential(vec![
|
||||
MockResponse::Complete(simple_text_events()),
|
||||
MockResponse::Hang(vec![]),
|
||||
]);
|
||||
let backend = Arc::new(
|
||||
WorkerRuntimeExecutionBackend::new(MockFactory {
|
||||
client: client.clone(),
|
||||
runtime_base: root.path().join("control"),
|
||||
cwd: root.path().to_path_buf(),
|
||||
store_dir: root.path().join("sessions"),
|
||||
worker_metadata_dir: root.path().join("metadata"),
|
||||
observed_cwds: Arc::default(),
|
||||
observed_workspace_clients: Arc::default(),
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||
crate::fs_store::FsRuntimeStoreOptions::new(root.path().join("catalog"))
|
||||
.with_runtime_id("delete-test"),
|
||||
backend.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test");
|
||||
let mut create = create_request("running Subject body");
|
||||
create.workspace_api = Some(WorkspaceApiRef {
|
||||
workspace_id: scope.workspace_id.clone(),
|
||||
base_url: "https://workspace.invalid".into(),
|
||||
});
|
||||
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||
workspace_id: scope.workspace_id.clone(),
|
||||
language: "English".into(),
|
||||
});
|
||||
let worker = runtime
|
||||
.create_worker_scoped(&scope, create.clone())
|
||||
.unwrap();
|
||||
runtime
|
||||
.send_input(&worker.worker_ref, WorkerInput::user("settled history"))
|
||||
.unwrap();
|
||||
let execution = backend.workers.lock().unwrap()[&worker.worker_ref].clone();
|
||||
backend
|
||||
.run_on_adapter_runtime({
|
||||
let handle = execution.handle.clone();
|
||||
let client = client.clone();
|
||||
async move {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while client.call_count.load(Ordering::SeqCst) < 1
|
||||
|| handle.shared_state.catalog_status() != WorkerStatus::Idle
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
let before = persisted_files(&root.path().join("sessions"));
|
||||
assert!(!before.is_empty());
|
||||
runtime
|
||||
.send_input(&worker.worker_ref, WorkerInput::user("run until stopped"))
|
||||
.unwrap();
|
||||
backend
|
||||
.run_on_adapter_runtime({
|
||||
let handle = execution.handle.clone();
|
||||
let client = client.clone();
|
||||
async move {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while client.call_count.load(Ordering::SeqCst) < 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Running);
|
||||
})
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: scope.workspace_id.clone(),
|
||||
worker_id: create.worker_id,
|
||||
create_fingerprint: create.create_fingerprint.clone(),
|
||||
operation_id: "delete-running".into(),
|
||||
};
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scope, request.clone())
|
||||
.unwrap();
|
||||
assert!(execution.handle.protocol_is_closed());
|
||||
assert!(execution.tasks.tasks.lock().unwrap().is_empty());
|
||||
assert!(backend.workers.lock().unwrap().is_empty());
|
||||
assert!(backend.pending_restores.lock().unwrap().is_empty());
|
||||
assert_eq!(
|
||||
runtime.worker_detail(&worker.worker_ref).unwrap().status,
|
||||
crate::catalog::WorkerStatus::Stopped
|
||||
);
|
||||
// Preserve the settled prefix. Ordinary Controller empty-turn rollback remains unchanged.
|
||||
let after = persisted_files(&root.path().join("sessions"));
|
||||
for (path, bytes) in before {
|
||||
let entries = |bytes: &[u8]| {
|
||||
bytes
|
||||
.split(|b| *b == b'\n')
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(|line| serde_json::from_slice::<serde_json::Value>(line).unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
assert!(
|
||||
entries(&after[&path]).starts_with(&entries(&bytes)),
|
||||
"{path:?} BEFORE={:?} AFTER={:?}",
|
||||
entries(&bytes),
|
||||
entries(&after[&path])
|
||||
);
|
||||
}
|
||||
assert!(runtime.create_worker_scoped(&scope, create).is_err());
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&worker.worker_ref,
|
||||
runtime.test_restore_request(&worker.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scope, request)
|
||||
.unwrap();
|
||||
assert_eq!(client.call_count.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
#[cfg(feature = "http-server")]
|
||||
fn subject_delete_unregistered_profile_controller_retries_unknown_shutdown_without_erasing_session()
|
||||
{
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (backend, request) = profile_stop_fixture(root.path());
|
||||
let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone();
|
||||
let deletion = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: request
|
||||
.workspace_scope
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.workspace_id
|
||||
.clone(),
|
||||
worker_id: request.worker_ref.worker_id,
|
||||
create_fingerprint: request.request.create_fingerprint.clone(),
|
||||
operation_id: "delete-unregistered".into(),
|
||||
};
|
||||
let mut bad = deletion.clone();
|
||||
bad.create_fingerprint.push_str("-wrong");
|
||||
assert!(backend.preflight_subject_delete_worker(&bad).is_err());
|
||||
assert!(backend.subject_delete_worker(&bad).is_err());
|
||||
let mut bad = deletion.clone();
|
||||
bad.workspace_id.push_str("-wrong");
|
||||
assert!(backend.subject_delete_worker(&bad).is_err());
|
||||
assert!(
|
||||
backend.admissions.lock().unwrap()[&request.worker_ref]
|
||||
.deletion
|
||||
.is_none()
|
||||
);
|
||||
let preparing = execution.clone();
|
||||
backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
prepare_rejected_shutdown(&preparing, false, None).await;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let session_root = root
|
||||
.path()
|
||||
.join("runtime/workers")
|
||||
.join(request.worker_ref.worker_id.to_string())
|
||||
.join("session");
|
||||
let before = persisted_files(&session_root);
|
||||
assert!(!before.is_empty());
|
||||
assert!(
|
||||
backend
|
||||
.subject_delete_worker(&deletion)
|
||||
.unwrap_err()
|
||||
.contains("Conflict")
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.workers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains_key(&request.worker_ref)
|
||||
);
|
||||
assert!(matches!(
|
||||
backend.restore_worker(request.clone()),
|
||||
WorkerExecutionSpawnResult::Rejected(_)
|
||||
));
|
||||
backend.subject_delete_worker(&deletion).unwrap();
|
||||
backend.subject_delete_worker(&deletion).unwrap();
|
||||
assert!(execution.handle.protocol_is_closed());
|
||||
assert!(execution.tasks.tasks.lock().unwrap().is_empty());
|
||||
assert!(backend.workers.lock().unwrap().is_empty());
|
||||
assert_eq!(persisted_files(&session_root), before);
|
||||
assert!(backend.preflight_restore(&request).is_err());
|
||||
assert!(
|
||||
backend
|
||||
.activate_restored_worker(request.operation_id, &request.worker_ref)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn subject_delete_pending_restore_fences_late_controller_and_never_restarts_it() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let restores = Arc::new(AtomicUsize::new(0));
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let handles = Arc::new(Mutex::new(Vec::new()));
|
||||
let backend = WorkerRuntimeExecutionBackend::new(LateControllerFactory {
|
||||
inner: MockFactory {
|
||||
client: MockClient::new(vec![]),
|
||||
runtime_base: root.path().join("control"),
|
||||
cwd: root.path().to_path_buf(),
|
||||
store_dir: root.path().join("sessions"),
|
||||
worker_metadata_dir: root.path().join("metadata"),
|
||||
observed_cwds: Arc::default(),
|
||||
observed_workspace_clients: Arc::default(),
|
||||
},
|
||||
restores: restores.clone(),
|
||||
release: release.clone(),
|
||||
actual_handles: handles.clone(),
|
||||
})
|
||||
.unwrap()
|
||||
.with_spawn_restore_timeout(Duration::from_millis(1));
|
||||
let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7()));
|
||||
request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new(
|
||||
"ws-test",
|
||||
"server-test",
|
||||
));
|
||||
request.request.workspace_api = Some(WorkspaceApiRef {
|
||||
workspace_id: "ws-test".into(),
|
||||
base_url: "https://workspace.invalid".into(),
|
||||
});
|
||||
assert!(matches!(
|
||||
backend.restore_worker(request.clone()),
|
||||
WorkerExecutionSpawnResult::ReconciliationRequired { .. }
|
||||
));
|
||||
let deletion = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: "ws-test".into(),
|
||||
worker_id: request.worker_ref.worker_id,
|
||||
create_fingerprint: request.request.create_fingerprint.clone(),
|
||||
operation_id: "delete-pending".into(),
|
||||
};
|
||||
// Unfinished factory ownership is unknown, never shutdown success.
|
||||
assert!(
|
||||
backend
|
||||
.subject_delete_worker(&deletion)
|
||||
.unwrap_err()
|
||||
.contains("pending")
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.pending_restores
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains_key(&request.worker_ref)
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.activate_restored_worker(request.operation_id, &request.worker_ref)
|
||||
.is_err()
|
||||
);
|
||||
release.notify_one();
|
||||
backend.subject_delete_worker(&deletion).unwrap();
|
||||
assert!(backend.pending_restores.lock().unwrap().is_empty());
|
||||
assert!(backend.workers.lock().unwrap().is_empty());
|
||||
assert_eq!(restores.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(handles.lock().unwrap().len(), 1);
|
||||
assert!(handles.lock().unwrap()[0].protocol_is_closed());
|
||||
assert!(matches!(
|
||||
backend.restore_worker(request),
|
||||
WorkerExecutionSpawnResult::Rejected(_)
|
||||
));
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
// Actual create factory future remains behind Runtime's Worker operation lock;
|
||||
// deletion must wait for that owner, then permanently reject the delayed retry.
|
||||
struct SubjectDeleteDeferredCreate {
|
||||
entered: mpsc::Sender<()>,
|
||||
release: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl RuntimeWorkerFactory for SubjectDeleteDeferredCreate {
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
_: WorkerExecutionRestoreRequest,
|
||||
) -> Result<Box<dyn RuntimeRestoreCandidate>, RuntimeRestoreFailure> {
|
||||
panic!("fenced create must never restore")
|
||||
}
|
||||
async fn spawn_controller(
|
||||
&self,
|
||||
_: WorkerExecutionSpawnRequest,
|
||||
) -> Result<RuntimeWorkerController, String> {
|
||||
self.entered.send(()).unwrap();
|
||||
self.release.notified().await;
|
||||
Err("create failed before any Controller".into())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_delete_serializes_inflight_create_and_rejects_all_late_admission() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (entered, entry) = mpsc::channel();
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let backend = Arc::new(
|
||||
WorkerRuntimeExecutionBackend::new(SubjectDeleteDeferredCreate {
|
||||
entered,
|
||||
release: release.clone(),
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("delete-test"),
|
||||
backend.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test");
|
||||
let mut create = create_request("inflight create");
|
||||
create.workspace_api = Some(WorkspaceApiRef {
|
||||
workspace_id: scope.workspace_id.clone(),
|
||||
base_url: "https://workspace.invalid".into(),
|
||||
});
|
||||
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||
workspace_id: scope.workspace_id.clone(),
|
||||
language: "English".into(),
|
||||
});
|
||||
let creating = {
|
||||
let runtime = runtime.clone();
|
||||
let scope = scope.clone();
|
||||
let create = create.clone();
|
||||
std::thread::spawn(move || runtime.create_worker_scoped(&scope, create))
|
||||
};
|
||||
entry.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: scope.workspace_id.clone(),
|
||||
worker_id: create.worker_id,
|
||||
create_fingerprint: create.create_fingerprint.clone(),
|
||||
operation_id: "delete-creating".into(),
|
||||
};
|
||||
let deleting = {
|
||||
let runtime = runtime.clone();
|
||||
let scope = scope.clone();
|
||||
let request = request.clone();
|
||||
std::thread::spawn(move || runtime.subject_delete_worker_scoped(&scope, request))
|
||||
};
|
||||
release.notify_one();
|
||||
assert!(creating.join().unwrap().is_err());
|
||||
deleting.join().unwrap().unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.create_worker_scoped(&scope, create.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(backend.workers.lock().unwrap().is_empty());
|
||||
let worker_ref = WorkerRef::new(create.worker_id);
|
||||
let spawn = WorkerExecutionSpawnRequest {
|
||||
worker_ref: worker_ref.clone(),
|
||||
request: create.clone(),
|
||||
workspace_scope: Some(scope.clone()),
|
||||
context: test_execution_context(worker_ref.clone()),
|
||||
workdir_attachments: BTreeMap::new(),
|
||||
config_bundle: None,
|
||||
};
|
||||
assert!(matches!(
|
||||
backend.spawn_worker(spawn),
|
||||
WorkerExecutionSpawnResult::Rejected(_)
|
||||
));
|
||||
let mut restore = direct_restore_request(worker_ref);
|
||||
restore.request = create;
|
||||
restore.workspace_scope = Some(scope);
|
||||
assert!(matches!(
|
||||
backend.restore_worker(restore),
|
||||
WorkerExecutionSpawnResult::Rejected(_)
|
||||
));
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
#[test]
|
||||
fn subject_delete_schema14_migration_adds_exclusion_store_without_changing_worker_or_receipts() {
|
||||
for with_fence in [false, true] {
|
||||
let (_temp, store, worker, path) = canonical_worker_store();
|
||||
let root = store.root.clone();
|
||||
let mut legacy =
|
||||
serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap();
|
||||
legacy["schema_version"] = serde_json::json!(RESTORE_REPAIR_SCHEMA_VERSION);
|
||||
fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap();
|
||||
let id = WorkerId::now_v7();
|
||||
let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id,
|
||||
"create_fingerprint": "fenced-create", "operation_id": "delete-1"});
|
||||
let mut response = request.clone();
|
||||
response["originals_absent"] = false.into();
|
||||
let fence =
|
||||
serde_json::json!({ id.to_string(): {"request": request, "response": response} });
|
||||
let mut runtime = serde_json::json!({"schema_version": RESTORE_REPAIR_SCHEMA_VERSION,
|
||||
"display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
|
||||
"workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [],
|
||||
"config_bundles": {}, "subject_create_recovery_fences": {}, "subject_releases": []});
|
||||
if with_fence {
|
||||
runtime["subject_delete_worker_fences"] = fence.clone();
|
||||
}
|
||||
fs::write(
|
||||
root.join(RUNTIME_FILE),
|
||||
serde_json::to_vec(&runtime).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
drop(store);
|
||||
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("test-runtime");
|
||||
let before = fs::read(&path).unwrap();
|
||||
assert!(
|
||||
FsRuntimeStore::migration_plan(&options)
|
||||
.unwrap()
|
||||
.migration_required
|
||||
);
|
||||
assert_eq!(fs::read(&path).unwrap(), before);
|
||||
let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
|
||||
legacy["schema_version"] = serde_json::json!(SCHEMA_VERSION);
|
||||
assert_eq!(
|
||||
read_json::<serde_json::Value>(&path, "read migrated Worker").unwrap(),
|
||||
legacy
|
||||
);
|
||||
let expected = if with_fence {
|
||||
fence
|
||||
} else {
|
||||
serde_json::json!({})
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(&opened.state.as_ref().unwrap().subject_delete_worker_fences)
|
||||
.unwrap(),
|
||||
expected
|
||||
);
|
||||
opened
|
||||
.store
|
||||
.write_runtime_snapshot(opened.state.as_ref().unwrap())
|
||||
.unwrap();
|
||||
drop(opened);
|
||||
let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(&reopened.state.unwrap().subject_delete_worker_fences).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_delete_store_rejects_missing_exclusion_field_malformed_receipt_and_workspace_owner() {
|
||||
for invalid in 0..4 {
|
||||
let (_temp, store, _worker, _path) = canonical_worker_store();
|
||||
let root = store.root.clone();
|
||||
let id = WorkerId::now_v7();
|
||||
let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id,
|
||||
"create_fingerprint": "fenced-create", "operation_id": "delete-1"});
|
||||
let mut response = request.clone();
|
||||
response["originals_absent"] = false.into();
|
||||
let mut runtime = serde_json::json!({"schema_version": SCHEMA_VERSION,
|
||||
"display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
|
||||
"workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [],
|
||||
"subject_delete_worker_fences": { id.to_string(): {"request": request, "response": response} }});
|
||||
match invalid {
|
||||
0 => {
|
||||
runtime
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("subject_delete_worker_fences");
|
||||
}
|
||||
1 => {
|
||||
runtime["subject_delete_worker_fences"][id.to_string()]["response"]["create_fingerprint"] =
|
||||
"different".into()
|
||||
}
|
||||
2 => runtime["workspace_owners"] = serde_json::json!({}),
|
||||
_ => {
|
||||
runtime["subject_delete_worker_fences"][id.to_string()]["request"]["worker_id"] =
|
||||
WorkerId::now_v7().to_string().into()
|
||||
}
|
||||
}
|
||||
let bytes = serde_json::to_vec(&runtime).unwrap();
|
||||
fs::write(root.join(RUNTIME_FILE), &bytes).unwrap();
|
||||
drop(store);
|
||||
assert!(FsRuntimeStore::open_or_create(root.clone(), "test-runtime").is_err());
|
||||
assert_eq!(fs::read(root.join(RUNTIME_FILE)).unwrap(), bytes);
|
||||
}
|
||||
}
|
||||
@@ -215,7 +215,8 @@ impl StoppedWorkerTemplate {
|
||||
snapshot
|
||||
.validate(&root.join(RUNTIME_FILE))
|
||||
.map_err(|_| reject("unsupported Runtime schema"))?;
|
||||
if !snapshot.subject_create_recovery_fences.is_empty()
|
||||
if !snapshot.subject_delete_worker_fences.is_empty()
|
||||
|| !snapshot.subject_create_recovery_fences.is_empty()
|
||||
|| !snapshot.subject_releases.is_empty()
|
||||
{
|
||||
return Err(reject("source has lifecycle operations"));
|
||||
@@ -427,6 +428,7 @@ impl StoppedWorkerTemplate {
|
||||
)?;
|
||||
self.validate_session(&session)?;
|
||||
store.write_runtime_snapshot(&PersistedRuntimeState {
|
||||
subject_delete_worker_fences: BTreeMap::new(),
|
||||
subject_create_recovery_fences: BTreeMap::new(),
|
||||
subject_releases: Vec::new(),
|
||||
// Match normal store initialization: this is catalog availability,
|
||||
|
||||
@@ -3415,6 +3415,14 @@ struct SubjectControllerReservation {
|
||||
state: runtime_api::SubjectReleaseState,
|
||||
}
|
||||
|
||||
// Admission identity is retained even when execution publication or stop settlement
|
||||
// failed. Runtime owns durable exclusion; this guards the adapter's delayed calls.
|
||||
struct WorkerAdmission {
|
||||
workspace_id: Option<String>,
|
||||
create_fingerprint: String,
|
||||
deletion: Option<runtime_api::SubjectDeleteWorkerRequest>,
|
||||
}
|
||||
|
||||
/// Worker execution backend backed by real worker crate Workers.
|
||||
pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
||||
#[cfg(test)]
|
||||
@@ -3438,6 +3446,7 @@ pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
||||
>,
|
||||
>,
|
||||
worker_locks: Mutex<HashMap<WorkerRef, Arc<Mutex<()>>>>,
|
||||
admissions: Mutex<HashMap<WorkerRef, WorkerAdmission>>,
|
||||
spawn_restore_timeout: Duration,
|
||||
subject_reservations: Mutex<HashMap<(WorkerRef, String), SubjectControllerReservation>>,
|
||||
shutdown_admission_timeout: Duration,
|
||||
@@ -3464,12 +3473,92 @@ where
|
||||
pending_restores: Mutex::new(HashMap::new()),
|
||||
completed_restore_failures: Mutex::new(HashMap::new()),
|
||||
worker_locks: Mutex::new(HashMap::new()),
|
||||
admissions: Mutex::new(HashMap::new()),
|
||||
spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT,
|
||||
subject_reservations: Mutex::new(HashMap::new()),
|
||||
shutdown_admission_timeout: Duration::from_secs(5),
|
||||
})
|
||||
}
|
||||
|
||||
// Caller holds the adapter Worker lock.
|
||||
fn admit_worker(
|
||||
&self,
|
||||
worker: &WorkerRef,
|
||||
workspace: Option<&str>,
|
||||
fingerprint: &str,
|
||||
) -> Result<(), String> {
|
||||
let mut admissions = self
|
||||
.admissions
|
||||
.lock()
|
||||
.map_err(|_| "worker admission lock poisoned")?;
|
||||
if let Some(saved) = admissions.get(worker) {
|
||||
if saved.deletion.is_some() {
|
||||
return Err("Subject Worker permanently excluded".into());
|
||||
}
|
||||
if saved.workspace_id.as_deref() != workspace || saved.create_fingerprint != fingerprint
|
||||
{
|
||||
return Err("Worker admission identity mismatch".into());
|
||||
}
|
||||
} else {
|
||||
admissions.insert(
|
||||
worker.clone(),
|
||||
WorkerAdmission {
|
||||
workspace_id: workspace.map(str::to_string),
|
||||
create_fingerprint: fingerprint.into(),
|
||||
deletion: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_subject_delete_identity(
|
||||
&self,
|
||||
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
let admissions = self
|
||||
.admissions
|
||||
.lock()
|
||||
.map_err(|_| "worker admission lock poisoned")?;
|
||||
if let Some(saved) = admissions.get(&WorkerRef::new(request.worker_id)) {
|
||||
if saved.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|
||||
|| saved.create_fingerprint != request.create_fingerprint
|
||||
|| saved
|
||||
.deletion
|
||||
.as_ref()
|
||||
.is_some_and(|saved| saved != request)
|
||||
{
|
||||
return Err("Subject delete execution identity mismatch".into());
|
||||
}
|
||||
}
|
||||
if let Some(pending) = self.pending_restore(&WorkerRef::new(request.worker_id))? {
|
||||
if pending
|
||||
.request
|
||||
.workspace_scope
|
||||
.as_ref()
|
||||
.map(|scope| scope.workspace_id.as_str())
|
||||
!= Some(request.workspace_id.as_str())
|
||||
|| pending.request.request.create_fingerprint != request.create_fingerprint
|
||||
{
|
||||
return Err("Subject delete pending execution identity mismatch".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_not_subject_deleted(&self, worker: &WorkerRef) -> Result<(), String> {
|
||||
if self
|
||||
.admissions
|
||||
.lock()
|
||||
.map_err(|_| "worker admission lock poisoned")?
|
||||
.get(worker)
|
||||
.is_some_and(|admission| admission.deletion.is_some())
|
||||
{
|
||||
return Err("Subject Worker permanently excluded".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn with_backend_id(mut self, backend_id: impl Into<String>) -> Self {
|
||||
self.backend_id = backend_id.into();
|
||||
self
|
||||
@@ -4739,6 +4828,58 @@ where
|
||||
&self.backend_id
|
||||
}
|
||||
|
||||
fn preflight_subject_delete_worker(
|
||||
&self,
|
||||
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
let worker = WorkerRef::new(request.worker_id);
|
||||
let lock = self.worker_lock(&worker)?;
|
||||
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
|
||||
self.validate_subject_delete_identity(request)
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
let worker = WorkerRef::new(request.worker_id);
|
||||
let lock = self.worker_lock(&worker)?;
|
||||
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
|
||||
self.validate_subject_delete_identity(request)?;
|
||||
self.admissions
|
||||
.lock()
|
||||
.map_err(|_| "worker admission lock poisoned")?
|
||||
.insert(
|
||||
worker.clone(),
|
||||
WorkerAdmission {
|
||||
workspace_id: Some(request.workspace_id.clone()),
|
||||
create_fingerprint: request.create_fingerprint.clone(),
|
||||
deletion: Some(request.clone()),
|
||||
},
|
||||
);
|
||||
let result = self.stop_worker_under_lock(&worker);
|
||||
if !result.is_accepted() {
|
||||
return Err(result.message_or_default());
|
||||
}
|
||||
// A live Controller and failed unpublished candidates can coexist. The
|
||||
// ordinary stop's live branch joins the Controller; also reconcile any
|
||||
// factory-owned cache/artifacts before issuing deletion proof.
|
||||
let factory = self.factory.clone();
|
||||
let stopped = worker.clone();
|
||||
self.run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move {
|
||||
factory.reconcile_stopped_worker(&stopped).await
|
||||
})?;
|
||||
self.subject_reservations
|
||||
.lock()
|
||||
.map_err(|_| "Subject reservations poisoned")?
|
||||
.retain(|(saved, _), _| saved != &worker);
|
||||
self.completed_restore_failures
|
||||
.lock()
|
||||
.map_err(|_| "completed restores poisoned")?
|
||||
.remove(&worker);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
request: &runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -5178,6 +5319,19 @@ where
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Err(message) = self.admit_worker(
|
||||
&request.worker_ref,
|
||||
request
|
||||
.workspace_scope
|
||||
.as_ref()
|
||||
.map(|scope| scope.workspace_id.as_str()),
|
||||
&request.request.create_fingerprint,
|
||||
) {
|
||||
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Spawn,
|
||||
message,
|
||||
));
|
||||
}
|
||||
match self.pending_restore(&request.worker_ref) {
|
||||
Ok(None) => {}
|
||||
Ok(Some(_)) => {
|
||||
@@ -5405,6 +5559,10 @@ where
|
||||
));
|
||||
}
|
||||
};
|
||||
self.ensure_not_subject_deleted(&request.worker_ref)
|
||||
.map_err(|message| {
|
||||
WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message)
|
||||
})?;
|
||||
if self
|
||||
.pending_restore(&request.worker_ref)
|
||||
.map_err(|message| {
|
||||
@@ -5499,6 +5657,19 @@ where
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Err(message) = self.admit_worker(
|
||||
&request.worker_ref,
|
||||
request
|
||||
.workspace_scope
|
||||
.as_ref()
|
||||
.map(|scope| scope.workspace_id.as_str()),
|
||||
&request.request.create_fingerprint,
|
||||
) {
|
||||
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Restore,
|
||||
message,
|
||||
));
|
||||
}
|
||||
if let Some(result) =
|
||||
self.completed_restore_failure(request.operation_id, &request.worker_ref)
|
||||
{
|
||||
@@ -6182,6 +6353,7 @@ where
|
||||
return Err(message);
|
||||
}
|
||||
};
|
||||
self.ensure_not_subject_deleted(worker_ref)?;
|
||||
if let Some(pending) = self.pending_restore(worker_ref)? {
|
||||
if pending.request.operation_id != operation_id {
|
||||
return Err("restored Worker operation identity does not match".into());
|
||||
@@ -6820,6 +6992,10 @@ mod tests {
|
||||
};
|
||||
use worker::PreparedWorker;
|
||||
|
||||
include!("subject_delete_backend_tests.rs");
|
||||
#[cfg(feature = "fs-store")]
|
||||
include!("subject_delete_create_tests.rs");
|
||||
|
||||
// Both candidate and failure suites need the same guarded, checkout-owned
|
||||
// temporary root, independently of Runtime's optional filesystem catalog.
|
||||
fn restore_failure_tempdir() -> tempfile::TempDir {
|
||||
|
||||
@@ -478,7 +478,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
(101, "Durable explicit Subject body release".into()),
|
||||
(102, "Durable failed Subject creation recovery".into()),
|
||||
(103, "Passkey metadata and atomic addition results".into()),
|
||||
(104, "Converge Passkey results and Job input retention".into())
|
||||
(104, "Converge Passkey results and Job input retention".into()),
|
||||
(105, "Durable Subject deletion snapshots and Job admission fences".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -1292,6 +1292,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
_request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||
Err("Subject deletion fencing is unavailable on this Runtime".into())
|
||||
}
|
||||
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
_request: runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -2269,6 +2276,21 @@ impl RuntimeRegistry {
|
||||
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
||||
}
|
||||
|
||||
pub fn subject_delete_worker(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
self.runtime(runtime_id)?
|
||||
.subject_delete_worker(request)
|
||||
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: runtime_id.into(),
|
||||
code: "subject_delete_worker_unproven".into(),
|
||||
message,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn recover_subject_create(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -3367,6 +3389,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||
if request.workspace_id != self.workspace_id {
|
||||
return Err("Subject deletion Workspace mismatch".into());
|
||||
}
|
||||
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
|
||||
self.runtime
|
||||
.subject_delete_worker_scoped(&scope, request)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -4255,7 +4290,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
|
||||
}
|
||||
if path.contains("/retention/")
|
||||
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|
||||
|| (path == "/v1/subject-create-recovery" && method == "POST")
|
||||
|| (matches!(
|
||||
path,
|
||||
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
|
||||
) && method == "POST")
|
||||
{
|
||||
return "workers:delete";
|
||||
}
|
||||
@@ -5656,6 +5694,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||
self.run_runtime_api(
|
||||
self.request_timeout,
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move { client.subject_delete_worker(request).await },
|
||||
)
|
||||
.map_err(|diagnostic| diagnostic.message)
|
||||
}
|
||||
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -8507,6 +8557,57 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_subject_delete_uses_generated_exact_identity_transport() {
|
||||
assert_eq!(
|
||||
workspace_runtime_operation("POST", "/v1/subject-delete-worker"),
|
||||
"workers:delete"
|
||||
);
|
||||
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: "workspace-test".into(),
|
||||
worker_id: EmbeddedWorkerId::now_v7(),
|
||||
create_fingerprint: "sha256:reserved-create".into(),
|
||||
operation_id: "delete-worker-1".into(),
|
||||
};
|
||||
let response = runtime_api::SubjectDeleteWorkerResponse {
|
||||
originals_absent: true,
|
||||
workspace_id: request.workspace_id.clone(),
|
||||
worker_id: request.worker_id,
|
||||
create_fingerprint: request.create_fingerprint.clone(),
|
||||
operation_id: request.operation_id.clone(),
|
||||
};
|
||||
// Only remote transport is substituted; actual Controller shutdown and
|
||||
// signed admission are covered by worker-runtime's independent suites.
|
||||
let (base_url, server) = serve_mock_http(vec![mock_response(
|
||||
"POST",
|
||||
"/v1/subject-delete-worker",
|
||||
true,
|
||||
200,
|
||||
serde_json::to_string(&response).unwrap(),
|
||||
)]);
|
||||
let registry = RuntimeRegistry::new(Vec::new());
|
||||
registry.register(
|
||||
RemoteWorkerRuntime::new(
|
||||
RemoteRuntimeConfig::new(
|
||||
"remote:delete",
|
||||
"Deletion transport fixture",
|
||||
base_url,
|
||||
Some("secret-token".into()),
|
||||
),
|
||||
"workspace-test".into(),
|
||||
"http://127.0.0.1:8787".into(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(
|
||||
registry
|
||||
.subject_delete_worker("remote:delete", request)
|
||||
.unwrap(),
|
||||
response
|
||||
);
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_subject_create_recovery_uses_generated_management_transport() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -367,14 +367,54 @@ async fn implementation_result_rejects_forged_evidence_and_rolls_back_without_no
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
|
||||
let (dir, store) = tracking_store_fixture().await;
|
||||
tracking_store_assign(&store, "delegation", "assign");
|
||||
let worker = routing_worker_assign(&store, "implementation").worker;
|
||||
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
|
||||
store.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event;
|
||||
// Preserve seeded assignment evidence while rebuilding the actual pre-business
|
||||
// capture composition. Changing only a receipt leaves schema105 admission fences
|
||||
// and erasure guards in an otherwise legacy fixture.
|
||||
fn reconstruct_implementation_schema96(conn: &Connection) -> Result<()> {
|
||||
assert_eq!(current_schema_version(conn)?, 105);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("retained96.db");
|
||||
prepare_retained_schema(&path, 96);
|
||||
let retained = Connection::open(path)?;
|
||||
for trigger in [
|
||||
"notification_sources_no_delete",
|
||||
"notification_decisions_no_delete",
|
||||
"notification_requests_no_delete",
|
||||
"notification_acknowledgements_no_delete",
|
||||
"notification_failures_no_delete",
|
||||
"notification_job_targets_no_delete",
|
||||
"notification_job_legacy_no_delete",
|
||||
"notification_source_preparation_failures_no_delete",
|
||||
] {
|
||||
let definition: String = retained.query_row(
|
||||
"SELECT sql FROM sqlite_schema WHERE type='trigger' AND name=?1",
|
||||
[trigger],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
conn.execute_batch(&format!("DROP TRIGGER {trigger}; {definition};"))?;
|
||||
}
|
||||
for trigger in [
|
||||
"subject_deleted_job_insert",
|
||||
"subject_deleted_job_result",
|
||||
"subject_deleted_attempt_insert",
|
||||
"subject_deleted_attempt_result",
|
||||
"subject_deleted_worker_reservation",
|
||||
"subject_deleted_notification_source",
|
||||
"subject_deleted_notification_payload",
|
||||
"subject_deleted_notification_send",
|
||||
] {
|
||||
conn.execute_batch(&format!("DROP TRIGGER {trigger};"))?;
|
||||
}
|
||||
// Children before parents; dropping the tables also drops snapshot guards.
|
||||
conn.execute_batch(
|
||||
"DROP TABLE subject_deletion_allocations;
|
||||
DROP TABLE subject_deletion_jobs;
|
||||
DROP TABLE subject_deletion_workers;
|
||||
DROP TABLE subject_deletions;
|
||||
UPDATE __yoi_schema_migrations SET version=104 WHERE version=105;",
|
||||
)?;
|
||||
conn.execute_batch(
|
||||
"DROP TRIGGER notification_mr_event;
|
||||
DROP TRIGGER notification_mr_source;
|
||||
DROP TRIGGER notification_implementation_result;
|
||||
DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
@@ -396,7 +436,43 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;",
|
||||
)?;
|
||||
assert_eq!(current_schema_version(conn)?, 96);
|
||||
assert_eq!(conn.query_row(
|
||||
"SELECT count(*) FROM sqlite_schema WHERE name LIKE 'subject_delet%' OR sql LIKE '%subject_deletion%'",
|
||||
[], |row| row.get::<_, i64>(0),
|
||||
)?, 0);
|
||||
for table in [
|
||||
"notification_sources",
|
||||
"notification_decisions",
|
||||
"notification_requests",
|
||||
"notification_acknowledgements",
|
||||
"notification_failures",
|
||||
"notification_job_targets",
|
||||
"notification_job_legacy_receipts",
|
||||
"notification_source_preparation_failures",
|
||||
"backend_jobs",
|
||||
"backend_job_attempts",
|
||||
"worker_create_reservations",
|
||||
] {
|
||||
assert_eq!(
|
||||
state_contract_schema_signature(conn, table),
|
||||
state_contract_schema_signature(&retained, table),
|
||||
"schema96 {table}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
|
||||
let (dir, store) = tracking_store_fixture().await;
|
||||
tracking_store_assign(&store, "delegation", "assign");
|
||||
let worker = routing_worker_assign(&store, "implementation").worker;
|
||||
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
|
||||
store.with_conn(|conn| {
|
||||
reconstruct_implementation_schema96(conn)?;
|
||||
migrate_business_notifications_v96_to_v97(conn)?;
|
||||
assert_eq!(current_schema_version(conn)?, 97);
|
||||
Ok(())
|
||||
@@ -522,24 +598,7 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
||||
store.with_conn(|conn| {
|
||||
conn.execute("INSERT INTO ticket_assignment_operation_fences VALUES('w','operation-known-implementation','delegation','r','worker','workspace-orchestrator','tracking-binding',?1)",[serde_json::to_string(&tracking_store_source()).unwrap()])?;
|
||||
// Reconstruct the prior composition before its new business capture existed.
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TRIGGER passkey_metadata_insert;
|
||||
DROP TABLE passkey_addition_receipts;
|
||||
DROP TABLE passkey_metadata;
|
||||
DROP TRIGGER backend_job_input_retirement;
|
||||
DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
||||
reconstruct_implementation_schema96(conn)?;
|
||||
conn.execute_batch("INSERT INTO merge_requests VALUES('w','historical-mr','main','open','refs/heads/work','refs/heads/main','3','3');
|
||||
INSERT INTO merge_request_ticket_relations VALUES('w','historical-mr','tracking','implements','3');
|
||||
INSERT INTO merge_request_thread_events VALUES('w','historical-mr','approved-event',1,'review','{\"subject_ref\":\"source-a\",\"decision\":\"approve\"}',NULL,'3');
|
||||
@@ -691,24 +750,7 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
||||
];
|
||||
store.with_conn(|conn| {
|
||||
if historical {
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TRIGGER passkey_metadata_insert;
|
||||
DROP TABLE passkey_addition_receipts;
|
||||
DROP TABLE passkey_metadata;
|
||||
DROP TRIGGER backend_job_input_retirement;
|
||||
DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
||||
reconstruct_implementation_schema96(conn)?;
|
||||
}
|
||||
conn.execute_batch("INSERT INTO merge_requests VALUES('w','actor-mr','main','open','work','main','3','3');
|
||||
INSERT INTO merge_request_ticket_relations VALUES('w','actor-mr','tracking','implements','3');")?;
|
||||
|
||||
@@ -1757,3 +1757,5 @@ WHEN (OLD.runtime_id != NEW.runtime_id OR OLD.worker_id != NEW.worker_id OR OLD.
|
||||
WHERE workspace_id = OLD.workspace_id AND runtime_id = OLD.runtime_id
|
||||
AND worker_id = OLD.worker_id)
|
||||
BEGIN SELECT RAISE(ABORT, 'Worker has unfinished Ticket work'); END;
|
||||
|
||||
-- Subject deletion schema is composed by store.rs.
|
||||
|
||||
@@ -444,6 +444,28 @@ fn job_result_facts(
|
||||
|
||||
pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) {
|
||||
use crate::store::JobNotificationFacts;
|
||||
// A recovered/captured result is not delivery authority. Serialize even the
|
||||
// Runtime send with deletion, so no prepared bytes escape after its fence.
|
||||
let subject = subjektiv_deletion::job_subject(&acceptance.job.request);
|
||||
let gate = match subject
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(api, subject))
|
||||
.transpose()
|
||||
{
|
||||
Ok(gate) => gate,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "Subject Job notification gate unavailable");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _guard = match gate.as_ref().map(|gate| gate.lock()).transpose() {
|
||||
Ok(guard) => guard,
|
||||
Err(_) => return,
|
||||
};
|
||||
if let Some(subject) = subject
|
||||
&& subjektiv_deletion::require_available(api, subject).is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -849,7 +849,16 @@ async fn notification_routing_fresh_and_upgraded_schema_and_guards_match() {
|
||||
let snapshot = |conn: &Connection| -> Vec<(String, String)> {
|
||||
conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::<rusqlite::Result<_>>().unwrap()
|
||||
};
|
||||
let upgraded = snapshot(&conn);
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 98);
|
||||
drop(conn);
|
||||
// Compare the complete current composition, not upgraded98 against fresh105.
|
||||
let upgraded_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let upgraded = upgraded_store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, 105);
|
||||
Ok(snapshot(conn))
|
||||
})
|
||||
.unwrap();
|
||||
let (fresh_directory, fresh) = tracking_store_fixture().await;
|
||||
fresh
|
||||
.with_conn(|conn| {
|
||||
|
||||
@@ -333,6 +333,15 @@ impl SqliteWorkspaceStore {
|
||||
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
|
||||
}
|
||||
};
|
||||
if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() {
|
||||
let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")?
|
||||
.query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
for id in prior {
|
||||
let prior=load_plan(&tx,&id)?.ok_or_else(||StoreError::Store("removal plan disappeared".into()))?;
|
||||
require_current_subject_deletion_plan(&tx,&prior)?;
|
||||
}
|
||||
}
|
||||
let mut blockers=Vec::new();
|
||||
if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); }
|
||||
if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? {
|
||||
@@ -363,6 +372,7 @@ impl SqliteWorkspaceStore {
|
||||
let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?;
|
||||
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
||||
if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));}
|
||||
require_current_subject_deletion_plan(&tx, &plan)?;
|
||||
if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);}
|
||||
if plan.state==WorkerRemovalPlanState::Blocked {
|
||||
return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));
|
||||
@@ -420,6 +430,8 @@ impl SqliteWorkspaceStore {
|
||||
WorkerRetentionError::Invalid("Worker id must be a canonical UUIDv7".to_string())
|
||||
})?;
|
||||
let removed_at = plan.created_at.clone();
|
||||
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
||||
.map_err(map_error)?;
|
||||
let prior_failure_category = plan.failure_category.clone();
|
||||
Ok(PreparedWorkerRemoval {
|
||||
runtime_request: WorkerRetentionExecutionRequest {
|
||||
@@ -459,8 +471,8 @@ impl SqliteWorkspaceStore {
|
||||
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
||||
)
|
||||
)
|
||||
ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END,
|
||||
created_at DESC LIMIT 1",
|
||||
ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN state='succeeded' THEN 1 ELSE 2 END,
|
||||
created_at DESC,operation_id DESC LIMIT 1",
|
||||
params![workspace_id, worker.runtime_id, worker.worker_id],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
@@ -474,6 +486,8 @@ impl SqliteWorkspaceStore {
|
||||
let Some(plan) = plan else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
||||
.map_err(map_error)?;
|
||||
let prior_failure_category = plan.failure_category.clone();
|
||||
let worker_id = plan
|
||||
.worker
|
||||
@@ -659,6 +673,7 @@ impl SqliteWorkspaceStore {
|
||||
let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?;
|
||||
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
||||
if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));}
|
||||
require_current_subject_deletion_plan(&tx, &plan)?;
|
||||
if plan.state==WorkerRemovalPlanState::Succeeded{
|
||||
// Repair projection fences left by the pre-atomic implementation and
|
||||
// return a replayable change for a broadcast missed after commit.
|
||||
@@ -1024,6 +1039,49 @@ fn load_removal_policy(
|
||||
let Some(mut policy) = load_policy(c, w)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Snapshot authority outranks normal policy and Journal cleanup policy. Body
|
||||
// and delegated Sessions must never inherit the ephemeral Job exception.
|
||||
if let Some(is_job) = crate::store::subject_deletion_worker_kind(c, w, worker)? {
|
||||
let update = WorkerRetentionPolicyUpdate {
|
||||
policy_id: if is_job {
|
||||
"subject-deletion-job-v1"
|
||||
} else {
|
||||
"subject-deletion-conversation-v1"
|
||||
}
|
||||
.into(),
|
||||
session_disposition: if is_job {
|
||||
SessionDisposition::Purge
|
||||
} else {
|
||||
SessionDisposition::Archive
|
||||
},
|
||||
metadata_disposition: MetadataDisposition::Tombstone,
|
||||
archive_retention: ArchiveRetention::Forever,
|
||||
// Runtime calls diagnostic archiving Retain, not Archive. Preserve
|
||||
// the Workspace's duration, or thirty days when it normally purges.
|
||||
diagnostics_disposition: if is_job {
|
||||
DiagnosticsDisposition::Purge
|
||||
} else {
|
||||
DiagnosticsDisposition::Retain
|
||||
},
|
||||
diagnostics_retention_seconds: if is_job {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
policy
|
||||
.diagnostics_retention_seconds
|
||||
.unwrap_or(30 * 24 * 60 * 60),
|
||||
)
|
||||
},
|
||||
};
|
||||
policy.policy_id = update.policy_id.clone();
|
||||
policy.policy_digest = worker_retention_policy_digest(&update);
|
||||
policy.session_disposition = update.session_disposition;
|
||||
policy.metadata_disposition = update.metadata_disposition;
|
||||
policy.archive_retention = update.archive_retention;
|
||||
policy.diagnostics_disposition = update.diagnostics_disposition;
|
||||
policy.diagnostics_retention_seconds = update.diagnostics_retention_seconds;
|
||||
return Ok(Some(policy));
|
||||
}
|
||||
if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? {
|
||||
let update = WorkerRetentionPolicyUpdate {
|
||||
policy_id: if erased_journal {
|
||||
@@ -1047,6 +1105,39 @@ fn load_removal_policy(
|
||||
}
|
||||
Ok(Some(policy))
|
||||
}
|
||||
fn require_current_subject_deletion_plan(
|
||||
c: &Connection,
|
||||
plan: &WorkerRemovalPlan,
|
||||
) -> crate::Result<()> {
|
||||
if plan.state == WorkerRemovalPlanState::Succeeded
|
||||
&& crate::store::subject_deletion_prior_removal_matches(
|
||||
c,
|
||||
&plan.workspace_id,
|
||||
&plan.worker,
|
||||
&plan.operation_id,
|
||||
)?
|
||||
{
|
||||
// An exact successful receipt pinned before Subject deletion owns its
|
||||
// independent original/archive lifecycle. Never retarget its policy.
|
||||
return Ok(());
|
||||
}
|
||||
if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() {
|
||||
let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false)?
|
||||
.ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?;
|
||||
if plan.session_disposition != current.session_disposition
|
||||
|| plan.metadata_disposition != current.metadata_disposition
|
||||
|| plan.archive_retention != current.archive_retention
|
||||
|| plan.diagnostics_disposition != current.diagnostics_disposition
|
||||
{
|
||||
return Err(stale_error(
|
||||
plan,
|
||||
"Subject deletion retention differs from saved plan; reconcile canonical Runtime removal before retry",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
||||
c.query_row(
|
||||
"SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition,
|
||||
@@ -2736,4 +2827,78 @@ mod tests {
|
||||
);
|
||||
assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() {
|
||||
let store = setup();
|
||||
let ordinary = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||
store
|
||||
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
||||
.unwrap();
|
||||
let body = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||
assert_eq!(body.session_disposition, SessionDisposition::Archive);
|
||||
assert_eq!(body.archive_retention, ArchiveRetention::Forever);
|
||||
assert_eq!(body.metadata_disposition, MetadataDisposition::Tombstone);
|
||||
assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain);
|
||||
assert!(body.diagnostics_retention_seconds.is_some());
|
||||
assert_ne!(body.policy_id, ordinary.policy_id);
|
||||
// A saved plan that would discard diagnostics cannot silently stand in
|
||||
// for the Subject conversation archive plan.
|
||||
assert!(matches!(
|
||||
store.begin_worker_removal("w", &ordinary.plan_id, &ordinary.input_fingerprint),
|
||||
Err(WorkerRetentionError::StalePlan { .. })
|
||||
));
|
||||
let ordinary_store = setup();
|
||||
assert_eq!(
|
||||
ordinary_store
|
||||
.plan_worker_removal(&req(), &inv())
|
||||
.unwrap()
|
||||
.policy_id,
|
||||
ordinary.policy_id
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_job_purge_rejects_recovery_of_old_executing_archive() {
|
||||
let store = setup();
|
||||
let old = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||
store
|
||||
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
|
||||
.unwrap();
|
||||
store
|
||||
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
||||
.unwrap();
|
||||
store
|
||||
.with_conn(|c| {
|
||||
c.execute(
|
||||
"UPDATE subject_deletion_workers SET is_job=1 WHERE workspace_id='w'",
|
||||
[],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
store.recover_worker_removal_execution("w", &req().worker),
|
||||
Err(WorkerRetentionError::StalePlan { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
store.plan_worker_removal(&req(), &inv()),
|
||||
Err(WorkerRetentionError::StalePlan { .. })
|
||||
));
|
||||
let fresh = store
|
||||
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(fresh.session_disposition, SessionDisposition::Purge);
|
||||
assert_eq!(fresh.diagnostics_disposition, DiagnosticsDisposition::Purge);
|
||||
assert_eq!(fresh.metadata_disposition, MetadataDisposition::Tombstone);
|
||||
assert_eq!(
|
||||
store
|
||||
.with_conn(|c| load_plan(c, &old.plan_id))
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.state,
|
||||
WorkerRemovalPlanState::Executing
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,10 +91,16 @@ mod schema_convergence {
|
||||
assert_eq!(plan.current_schema_version, 103);
|
||||
assert_eq!(
|
||||
plan.migrations,
|
||||
vec![WorkspaceSchemaMigrationStep {
|
||||
vec![
|
||||
WorkspaceSchemaMigrationStep {
|
||||
version: 104,
|
||||
name: "Converge Passkey results and Job input retention".into(),
|
||||
}]
|
||||
},
|
||||
WorkspaceSchemaMigrationStep {
|
||||
version: 105,
|
||||
name: "Durable Subject deletion snapshots and Job admission fences".into(),
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
schema_rows(&conn),
|
||||
@@ -109,6 +115,7 @@ mod schema_convergence {
|
||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||
let mut expected = history.clone();
|
||||
expected.push((104, "Converge Passkey results and Job input retention".into()));
|
||||
expected.push((105, "Durable Subject deletion snapshots and Job admission fences".into()));
|
||||
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
||||
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
|
||||
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
|
||||
@@ -138,7 +145,7 @@ mod schema_convergence {
|
||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||
assert_eq!(
|
||||
workspace_schema_migration_history(conn)?.len(),
|
||||
history.len() + 1
|
||||
history.len() + 2
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
@@ -176,7 +183,7 @@ mod schema_convergence {
|
||||
upgraded
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||
assert_eq!(current_schema_version(conn)?, 104);
|
||||
assert_eq!(current_schema_version(conn)?, 105);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
@@ -28,6 +28,7 @@ use subjektiv::api::*;
|
||||
mod backend_job_profile;
|
||||
mod external_operation_audit;
|
||||
mod subject_authorization;
|
||||
mod subjektiv_deletion;
|
||||
mod subjektiv_job;
|
||||
mod subjektiv_journal;
|
||||
mod subjektiv_journal_browser;
|
||||
@@ -4577,6 +4578,12 @@ impl WorkspaceApi {
|
||||
.run(move || recover_workdir_removals(&workdir_recovery_api))
|
||||
.await?;
|
||||
recover_runtime_removals(&api).await;
|
||||
let deletion_api = api.clone();
|
||||
let deletion_runtime = tokio::runtime::Handle::current();
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || deletion_runtime.block_on(subjektiv_deletion::recover(&deletion_api)))
|
||||
.await?;
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
@@ -4604,9 +4611,23 @@ impl WorkspaceApi {
|
||||
) -> Result<BackendJobReservation> {
|
||||
self.resolve_backend_job_profile(request)?;
|
||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let reservation =
|
||||
let reservation = {
|
||||
let gate = subjektiv_deletion::job_subject(request)
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let _guard = gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subjektiv_deletion::job_subject(request) {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
self.store
|
||||
.reserve_backend_job(&self.config.workspace_id, request, &now)?;
|
||||
.reserve_backend_job(&self.config.workspace_id, request, &now)?
|
||||
};
|
||||
let result = self.dispatch_backend_job_reservation(reservation);
|
||||
if result.is_err() {
|
||||
self.drain_reserved_backend_jobs();
|
||||
@@ -4621,9 +4642,27 @@ impl WorkspaceApi {
|
||||
/// immutable request.
|
||||
pub fn retry_backend_job(&self, job_id: &str) -> Result<BackendJobReservation> {
|
||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let reservation =
|
||||
let reservation = {
|
||||
let current = self
|
||||
.store
|
||||
.get_backend_job(self.workspace_id(), job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("unknown Backend Job".into()))?;
|
||||
let gate = subjektiv_deletion::job_subject(¤t.request)
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let _guard = gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subjektiv_deletion::job_subject(¤t.request) {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
self.store
|
||||
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?;
|
||||
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?
|
||||
};
|
||||
let result = self.dispatch_backend_job_reservation(reservation);
|
||||
if result.is_err() {
|
||||
self.drain_reserved_backend_jobs();
|
||||
@@ -4637,13 +4676,8 @@ impl WorkspaceApi {
|
||||
) -> Result<BackendJobReservation> {
|
||||
// Reread even a terminal/stale reservation before deciding whether it can
|
||||
// dispatch or replay: an explicit erasure may have retired its saved input.
|
||||
let journal_gate = reservation
|
||||
.job
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id))
|
||||
let journal_gate = subjektiv_deletion::job_subject(&reservation.job.request)
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let _journal_guard = journal_gate
|
||||
.as_ref()
|
||||
@@ -4652,6 +4686,9 @@ impl WorkspaceApi {
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subjektiv_deletion::job_subject(&reservation.job.request) {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
let current_job = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
|
||||
@@ -5571,6 +5608,27 @@ impl WorkspaceApi {
|
||||
source_worker: &RuntimeWorkerRef,
|
||||
submission: &BackendJobResultSubmission,
|
||||
) -> Result<BackendJobResultAcceptance> {
|
||||
let result_job = self
|
||||
.store
|
||||
.get_backend_job(self.workspace_id(), &submission.job_id)?;
|
||||
let subject_gate = result_job
|
||||
.as_ref()
|
||||
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let subject_guard = subject_gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = result_job
|
||||
.as_ref()
|
||||
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
|
||||
{
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let mut verified_submission = submission.clone();
|
||||
if let Some(job) = self
|
||||
@@ -5696,6 +5754,7 @@ impl WorkspaceApi {
|
||||
&verified_submission,
|
||||
&accepted_at,
|
||||
)?;
|
||||
drop(subject_guard);
|
||||
if let Some(job) = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
|
||||
@@ -5899,15 +5958,21 @@ impl WorkspaceApi {
|
||||
.map(|assignment| assignment.operation_id.clone())
|
||||
})
|
||||
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
|
||||
let subject = singleton_key
|
||||
let body_subject = singleton_key
|
||||
.as_deref()
|
||||
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
|
||||
if let Some(subject) = subject
|
||||
if let Some(subject) = body_subject
|
||||
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
|
||||
{
|
||||
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
|
||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||
}
|
||||
let subject = body_subject.or_else(|| {
|
||||
request
|
||||
.resolved_execution_grant
|
||||
.as_ref()
|
||||
.map(|grant| grant.subject_id.as_str())
|
||||
});
|
||||
let subject_gate = subject
|
||||
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
|
||||
.transpose()?;
|
||||
@@ -5919,6 +5984,9 @@ impl WorkspaceApi {
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subject {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
if let Some(subject) = body_subject {
|
||||
subjektiv_journal::check_start_allowed(self, subject)?;
|
||||
}
|
||||
let reservation = match self.config_store.reserve_worker_create(
|
||||
@@ -5943,11 +6011,11 @@ impl WorkspaceApi {
|
||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||
}
|
||||
};
|
||||
drop(subject_guard);
|
||||
let worker_id = reservation.worker_id;
|
||||
if let Some(grant) = request.resolved_execution_grant.as_ref() {
|
||||
subject_authorization::bind_reserved_grant(self, grant, &worker_id.to_string())?;
|
||||
}
|
||||
drop(subject_guard);
|
||||
// A completed exact-source allocation is historical acceptance, not a new
|
||||
// launch. Source/grant authorization and the full fingerprint were checked
|
||||
// above. Do not reserve its already-linked attachments again, redispatch
|
||||
@@ -7458,7 +7526,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou
|
||||
.merge(server_api::server_api_axum::subjektiv_memory_changes(
|
||||
service.clone(),
|
||||
))
|
||||
.merge(server_api::server_api_axum::subjektiv_subject_retire(
|
||||
.merge(server_api::server_api_axum::subjektiv_subject_delete(
|
||||
service.clone(),
|
||||
))
|
||||
.merge(server_api::server_api_axum::subjektiv_stage_candidate(
|
||||
@@ -9724,21 +9792,24 @@ impl server_api::ServerApi for ServerApiContractService {
|
||||
.map_err(ApiError::into_repository_api_error)
|
||||
}
|
||||
|
||||
async fn subjektiv_subject_retire(
|
||||
async fn subjektiv_subject_delete(
|
||||
&self,
|
||||
context: server_api::ServerRequestContext,
|
||||
workspace_id: String,
|
||||
subject_id: String,
|
||||
) -> std::result::Result<server_api::SubjektivSubjectResponse, server_api::RepositoryApiError>
|
||||
{
|
||||
scoped_retire_subjektiv_subject(
|
||||
State(self.workspace_api()?.clone()),
|
||||
AxumPath(ScopedSubjektivSubjectPath {
|
||||
workspace_id,
|
||||
subject_id,
|
||||
}),
|
||||
)
|
||||
request: server_api::SubjektivSubjectDeleteRequest,
|
||||
) -> std::result::Result<
|
||||
server_api::SubjektivSubjectDeleteResponse,
|
||||
server_api::RepositoryApiError,
|
||||
> {
|
||||
let api = self.workspace_api()?.clone();
|
||||
let actor = subject_authorization::administration_actor(&context)
|
||||
.map_err(ApiError::into_repository_api_error)?;
|
||||
require_workspace_owner(&api, &workspace_id, actor, "Subject deletion")
|
||||
.await
|
||||
.map_err(ApiError::into_repository_api_error)?;
|
||||
subjektiv_deletion::delete(api, workspace_id, subject_id, request)
|
||||
.await
|
||||
.map(|Json(response)| response)
|
||||
.map_err(ApiError::into_repository_api_error)
|
||||
}
|
||||
|
||||
@@ -22674,7 +22745,9 @@ fn subjektiv_subject_response(
|
||||
memory_fingerprint,
|
||||
state: match subject.state {
|
||||
crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active,
|
||||
crate::subjektiv::SubjectState::Retired => server_api::SubjektivSubjectState::Retired,
|
||||
crate::subjektiv::SubjectState::LegacyRetired => {
|
||||
server_api::SubjektivSubjectState::LegacyRetired
|
||||
}
|
||||
},
|
||||
created_at: subject.created_at,
|
||||
updated_at: subject.updated_at,
|
||||
@@ -22962,24 +23035,6 @@ async fn scoped_list_subjektiv_memory_changes(
|
||||
.await
|
||||
}
|
||||
|
||||
async fn scoped_retire_subjektiv_subject(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedSubjektivSubjectPath>,
|
||||
) -> ApiResult<Json<server_api::SubjektivSubjectResponse>> {
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
let subject = open_subjektiv_store(&api)?
|
||||
.retire_subject(&path.subject_id)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
// Retirement deliberately does not stop or remove the current Worker. The
|
||||
// Worker lease and subject lifecycle are independent authorities.
|
||||
Ok(Json(subjektiv_subject_response(&api, subject)?))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn scoped_stage_subjektiv_candidate(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedWorkspacePath>,
|
||||
@@ -24564,6 +24619,7 @@ fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error {
|
||||
Error::WorkspacePermissionDenied(error.to_string())
|
||||
}
|
||||
crate::subjektiv::SubjektivError::SubjectPermissionsConflict(_)
|
||||
| crate::subjektiv::SubjektivError::SubjectDeletionConflict(_)
|
||||
| crate::subjektiv::SubjektivError::PermissionOperationConflict(_)
|
||||
| crate::subjektiv::SubjektivError::WorkerExecutionGrantConflict(_) => {
|
||||
Error::RepositoryConflict(error.to_string())
|
||||
@@ -31543,6 +31599,7 @@ async fn workspace_response(
|
||||
manage_secrets: is_owner,
|
||||
manage_runtimes: is_owner,
|
||||
delete_workspace: is_owner,
|
||||
delete_subjects: is_owner,
|
||||
},
|
||||
extension_points: WorkspaceExtensionPoints {
|
||||
store: "sqlite".to_string(),
|
||||
@@ -37777,6 +37834,7 @@ mod tests {
|
||||
mod restore_workdir_tests;
|
||||
mod restored_parent_control_tests;
|
||||
mod subject_authorization_tests;
|
||||
mod subject_deletion_tests;
|
||||
mod subject_spawn_tests;
|
||||
mod subjektiv_jobs_tests;
|
||||
mod subjektiv_journal_tests;
|
||||
@@ -46241,7 +46299,13 @@ mod tests {
|
||||
let retired = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("retired").unwrap())
|
||||
.unwrap();
|
||||
store.retire_subject(&retired.id).unwrap();
|
||||
// Frozen lifecycle fixture: migration preserves old inactive Subjects; there is
|
||||
// intentionally no public operation that creates this historical state anymore.
|
||||
api.feature_storage.open(&api.subjektiv_registration).unwrap()
|
||||
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute("UPDATE subjects SET state='retired', record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&retired.id])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
|
||||
let request = |subject_id: Option<&str>, profile: &str, singleton_key: Option<&str>| {
|
||||
CreateWorkspaceWorkerRequest {
|
||||
@@ -48843,6 +48907,7 @@ mod tests {
|
||||
>,
|
||||
materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
|
||||
spawn_failure: std::sync::Mutex<Option<String>>,
|
||||
subject_delete_failure: std::sync::Mutex<Option<String>>,
|
||||
recovery_failure: std::sync::Mutex<Option<String>>,
|
||||
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
|
||||
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
|
||||
@@ -48887,6 +48952,7 @@ mod tests {
|
||||
std::env::temp_dir().join(unique),
|
||||
),
|
||||
spawn_failure: std::sync::Mutex::new(None),
|
||||
subject_delete_failure: std::sync::Mutex::new(None),
|
||||
recovery_failure: std::sync::Mutex::new(None),
|
||||
recovery_requests: std::sync::Mutex::new(Vec::new()),
|
||||
recovery_hook: std::sync::Mutex::new(None),
|
||||
@@ -49242,6 +49308,24 @@ mod tests {
|
||||
.map(|execution| execution.snapshot())
|
||||
}
|
||||
|
||||
fn preflight_subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> std::result::Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> std::result::Result<(), String> {
|
||||
if let Some(message) = self.subject_delete_failure.lock().unwrap().clone() {
|
||||
return Err(message);
|
||||
}
|
||||
self.stop_worker(&worker_runtime::identity::WorkerRef::new(request.worker_id));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stop_worker(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
|
||||
@@ -149,6 +149,13 @@ pub(super) fn worker_permissions(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ApiResult<Option<Vec<ScopedPermission>>> {
|
||||
if subjektiv_deletion::deleting_worker(api, worker)?
|
||||
|| open_subjektiv_store(api)?
|
||||
.worker_subject_deleted(&worker.runtime_id, &worker.worker_id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
{
|
||||
return Err(denied("Worker belonged to an explicitly deleted Subject"));
|
||||
}
|
||||
if let Some(grant) = live_child_grant(api, worker)? {
|
||||
return Ok(Some(grant.permissions));
|
||||
}
|
||||
@@ -226,6 +233,8 @@ pub(super) fn prepare_delegation(
|
||||
singleton_key: Option<&str>,
|
||||
subject_connection: bool,
|
||||
) -> ApiResult<Option<WorkerExecutionGrant>> {
|
||||
// A deleted grant must never become an ordinary non-Subject delegation source.
|
||||
worker_permissions(api, source)?;
|
||||
let child = live_child_grant(api, source)?;
|
||||
let body = if child.is_none() {
|
||||
current_body_subject(api, source)?
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
//! Explicit Subject destruction is independent of ordinary Worker retention. The domain
|
||||
//! fence commits before external effects, remains after failures, and rejects cached grants.
|
||||
//! Worker originals are archived; Job execution copies are purged. Existing archives, Workdirs,
|
||||
//! Tickets and other Subjects are not erased. Archive contents can include previously injected
|
||||
//! Memory: their independent retention/access policy still applies. This is logical content
|
||||
//! deletion, not a promise to erase backups, external providers or already exported material.
|
||||
//!
|
||||
//! Child removal/create-fence receipts, not acceptance, Idle or missing projections, prove
|
||||
//! quiescence. A retry carries the same scope/Subject/operation; no new child identity is issued.
|
||||
//! The domain receipt is authoritative completion. Backend inventories retain only identifiers
|
||||
//! needed for cleanup and fail-closed authorization, never a copy of Subject content.
|
||||
use super::*;
|
||||
use rusqlite::params;
|
||||
|
||||
pub(super) fn job_subject(request: &BackendJobRequest) -> Option<&str> {
|
||||
request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.map(|g| g.subject_id.as_str())
|
||||
.or_else(|| {
|
||||
request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_ref()
|
||||
.map(|g| g.subject_id.as_str())
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn require_available(api: &WorkspaceApi, subject: &str) -> Result<()> {
|
||||
let store = open_subjektiv_store(api).map_err(|error| error.error)?;
|
||||
if store
|
||||
.subject(subject)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::SubjektivSubjectNotFound(subject.into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn pending(api: &WorkspaceApi, subject: &str) -> Result<bool> {
|
||||
api.config_store.with_conn(|connection| {
|
||||
Ok(connection.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2)",
|
||||
params![api.workspace_id(), subject], |row| row.get(0),
|
||||
)?)
|
||||
})
|
||||
}
|
||||
|
||||
/// This is internal cleanup evidence, never a user-supplied removal flag. It exists only
|
||||
/// after the independently durable domain fence and exact-target inventory commit.
|
||||
pub(super) fn deleting_worker(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<bool> {
|
||||
api.config_store.with_conn(|connection| {
|
||||
Ok(connection.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM subject_deletion_workers WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)",
|
||||
params![api.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get(0),
|
||||
)?)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn delete(
|
||||
api: WorkspaceApi,
|
||||
workspace: String,
|
||||
subject: String,
|
||||
request: server_api::SubjektivSubjectDeleteRequest,
|
||||
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
validate_workspace_scope(&api, &workspace)?;
|
||||
handle.block_on(resume(&api, &subject, &request.operation_id))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn recover(api: &WorkspaceApi) -> Result<()> {
|
||||
let pending = open_subjektiv_store(api)
|
||||
.map_err(|error| error.error)?
|
||||
.pending_subject_deletions()
|
||||
.map_err(|e| Error::Store(e.to_string()))?;
|
||||
for deletion in pending {
|
||||
if let Err(error) = resume(api, &deletion.subject_id, &deletion.operation_id).await {
|
||||
tracing::warn!(%error.error, subject_id=%deletion.subject_id,
|
||||
"Subject deletion restart recovery remains incomplete; same request can resume");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resume(
|
||||
api: &WorkspaceApi,
|
||||
subject: &str,
|
||||
operation: &str,
|
||||
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
|
||||
let store = open_subjektiv_store(api)?;
|
||||
let gate = subjektiv_journal::lifecycle_lock(api, subject)?;
|
||||
let receipt = {
|
||||
let _guard = gate
|
||||
.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
|
||||
let receipt = store
|
||||
.begin_subject_deletion(subject, operation)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
if !receipt.completed {
|
||||
let delegated = store
|
||||
.deletion_subject_workers(subject)
|
||||
.map_err(subjektiv_store_error)?
|
||||
.into_iter()
|
||||
.map(|(runtime, worker)| RuntimeWorkerRef::new(runtime, worker))
|
||||
.collect::<Vec<_>>();
|
||||
api.config_store.prepare_subject_deletion(
|
||||
api.workspace_id(),
|
||||
subject,
|
||||
operation,
|
||||
&delegated,
|
||||
)?;
|
||||
}
|
||||
receipt
|
||||
};
|
||||
let response = |completed| server_api::SubjektivSubjectDeleteResponse {
|
||||
subject_id: subject.into(),
|
||||
operation_id: operation.into(),
|
||||
completed,
|
||||
};
|
||||
if receipt.completed {
|
||||
return Ok(response(true));
|
||||
}
|
||||
// Never hold the Host callback gate across a Runtime call. A create/restore can be
|
||||
// waiting for the callback that now has to reject this permanently fenced Subject.
|
||||
let workers =
|
||||
api.config_store
|
||||
.list_subject_deletion_workers(api.workspace_id(), subject, operation)?;
|
||||
for (worker, _is_job) in workers {
|
||||
if let Err(error) = remove_worker(api, subject, operation, &worker).await {
|
||||
tracing::warn!(%error.error, subject_id=subject, operation_id=operation, "Subject deletion remains incomplete; exact retry required");
|
||||
return Ok(response(false));
|
||||
}
|
||||
}
|
||||
{
|
||||
let _guard = gate
|
||||
.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
|
||||
api.config_store
|
||||
.erase_subject_deleted_jobs(api.workspace_id(), subject, operation)?;
|
||||
// Finish Backend cleanup before the authoritative domain receipt. If domain
|
||||
// purge fails, its separately committed fence and pending operation survive.
|
||||
// No retry can report domain completion while occupancy/Job cleanup is unproven.
|
||||
api.config_store
|
||||
.finish_subject_deletion_record(api.workspace_id(), subject, operation)?;
|
||||
store
|
||||
.finish_subject_deletion(subject, operation)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
}
|
||||
Ok(response(true))
|
||||
}
|
||||
|
||||
async fn remove_worker(
|
||||
api: &WorkspaceApi,
|
||||
subject: &str,
|
||||
operation: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ApiResult<()> {
|
||||
let request = api.config_store.subject_deletion_worker_request(
|
||||
api.workspace_id(),
|
||||
subject,
|
||||
operation,
|
||||
worker,
|
||||
)?;
|
||||
let fence = api
|
||||
.runtime
|
||||
.subject_delete_worker(&worker.runtime_id, request.clone())
|
||||
.map_err(RuntimeRegistryError::into_error)?;
|
||||
if fence.workspace_id != request.workspace_id
|
||||
|| fence.worker_id != request.worker_id
|
||||
|| fence.create_fingerprint != request.create_fingerprint
|
||||
|| fence.operation_id != request.operation_id
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Subject Worker deletion fence receipt mismatch".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
// Replay the canonical child even when physical removal already happened:
|
||||
// its retained receipt can finish the Backend catalog midpoint. The Runtime
|
||||
// fence above also prevents an older create from arriving after that removal.
|
||||
if api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(api.workspace_id(), worker)
|
||||
.map_err(|error| Error::Store(error.to_string()))?
|
||||
.is_some()
|
||||
{
|
||||
return execute_removal(api, operation, worker).await;
|
||||
}
|
||||
match api.runtime.worker(worker) {
|
||||
Ok(detail) => {
|
||||
if !detail.state.eq_ignore_ascii_case("stopped") {
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(
|
||||
worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(format!("Explicit Subject deletion {operation}")),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(RuntimeRegistryError::into_error)?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Subject Worker stop was not accepted".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
let actual = api
|
||||
.runtime
|
||||
.worker(worker)
|
||||
.map_err(RuntimeRegistryError::into_error)?;
|
||||
if !actual.state.eq_ignore_ascii_case("stopped") {
|
||||
return Err(
|
||||
Error::RepositoryConflict("Subject Worker stop is unconfirmed".into()).into(),
|
||||
);
|
||||
}
|
||||
if api
|
||||
.store
|
||||
.get_worker_registry(api.workspace_id(), worker)?
|
||||
.is_none()
|
||||
{
|
||||
// The Runtime fence authenticated this exact reserved fingerprint. A create
|
||||
// can have reached Runtime before its Backend publication failed. Publish
|
||||
// only its stopped record so the normal archive/Purge owner can remove it.
|
||||
record_worker_summary(
|
||||
api,
|
||||
&actual,
|
||||
&actual.display_name,
|
||||
actual.profile.clone(),
|
||||
WorkerRegistryDisplayNamePolicy::PreserveExisting,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Err(RuntimeRegistryError::UnknownWorker { .. }) => {
|
||||
if api
|
||||
.store
|
||||
.get_worker_registry(api.workspace_id(), worker)?
|
||||
.is_some()
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Runtime object missing before canonical original archival".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
api.config_store.confirm_subject_deletion_absence(
|
||||
api.workspace_id(),
|
||||
subject,
|
||||
operation,
|
||||
worker,
|
||||
&fence,
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => return Err(error.into_error().into()),
|
||||
}
|
||||
execute_removal(api, operation, worker).await
|
||||
}
|
||||
|
||||
async fn execute_removal(
|
||||
api: &WorkspaceApi,
|
||||
operation: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ApiResult<()> {
|
||||
let response = WorkerRemovalService::new(api)
|
||||
.execute_target_removal_on_host(
|
||||
api.runtime.as_ref(),
|
||||
worker,
|
||||
&format!("Explicit Subject deletion {operation}"),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::Store)?;
|
||||
if response.status != StatusCode::OK.as_u16() {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Subject Worker archival/Purge remains unconfirmed".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -446,6 +446,7 @@ pub(super) fn bind_host_session(
|
||||
/// Runs under the same Subject gate as final revoke/singleton release. A failed create or
|
||||
/// legacy removal may have released its Backend reservation but cannot hide a durable writer.
|
||||
pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<()> {
|
||||
subjektiv_deletion::require_available(api, subject)?;
|
||||
let store = store(api)?;
|
||||
let key = crate::subjektiv::subject_worker_singleton_key(subject)
|
||||
.map_err(|e| Error::InvalidInput(e.to_string()))?;
|
||||
@@ -1747,6 +1748,13 @@ pub(super) fn prepare_worker_removal(
|
||||
runtime: &RuntimeRegistry,
|
||||
target: &RuntimeWorkerRef,
|
||||
) -> Result<Option<crate::retention::ErasedJournalSession>> {
|
||||
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||
{
|
||||
// Full Subject deletion has a permanent domain fence. It deliberately drops
|
||||
// Journal continuity, but retains source originals through its archive policy.
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(attr) = removal_attribution(service, target)? else {
|
||||
if service
|
||||
.store
|
||||
@@ -1809,6 +1817,13 @@ pub(super) fn erased_archive_removal_proof(
|
||||
target: &RuntimeWorkerRef,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
|
||||
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||
{
|
||||
// Resume the exact canonical retention request after full Subject fencing.
|
||||
// Subject deletion retains originals; it is not per-Session source erasure.
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(attr) = removal_attribution(service, target)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -1852,6 +1867,13 @@ pub(super) fn validate_removal_retention(
|
||||
target: &RuntimeWorkerRef,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
) -> Result<()> {
|
||||
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||
{
|
||||
// The plan-local retention policy and canonical Runtime receipt remain required.
|
||||
// Neither unfinished Journals nor a stopped old Controller are deletion blockers.
|
||||
return Ok(());
|
||||
}
|
||||
let Some(attr) = removal_attribution(service, target)? else {
|
||||
if service
|
||||
.store
|
||||
@@ -2066,6 +2088,9 @@ pub(super) fn session_erased(api: &WorkspaceApi, request: &BackendJobRequest) ->
|
||||
let Some(grant) = &request.grants.subjektiv_journal else {
|
||||
return Ok(false);
|
||||
};
|
||||
if subjektiv_deletion::pending(api, &grant.subject_id)? {
|
||||
return Ok(true);
|
||||
}
|
||||
let store = store(api)?;
|
||||
store
|
||||
.journal_session_erased(&host(api, &store, grant)?)
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
//! Real router/authentication, feature store and Backend DB boundaries. Runtime
|
||||
//! execution/unknown and restart fences are covered in worker-runtime and the
|
||||
//! canonical removal suites, rather than claiming HTTP acceptance proves shutdown.
|
||||
use super::*;
|
||||
|
||||
async fn delete_request(
|
||||
api: &WorkspaceApi,
|
||||
subject: &str,
|
||||
key: &str,
|
||||
token: Option<&str>,
|
||||
workspace: &str,
|
||||
) -> (StatusCode, Value) {
|
||||
let mut builder = Request::builder()
|
||||
.method("DELETE")
|
||||
.uri(format!("/api/w/{workspace}/subjektiv/subjects/{subject}"))
|
||||
.header(CONTENT_TYPE, "application/json");
|
||||
if let Some(token) = token {
|
||||
builder = builder.header("authorization", format!("Bearer {token}"));
|
||||
}
|
||||
let response = build_router(api.clone())
|
||||
.oneshot(
|
||||
builder
|
||||
.body(Body::from(json!({"operation_id":key}).to_string()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let bytes = to_bytes(response.into_body(), 2 * 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = serde_json::from_slice(&bytes)
|
||||
.unwrap_or_else(|_| json!({"text":String::from_utf8_lossy(&bytes)}));
|
||||
(status, body)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deletion_owner_scope_exact_retry_and_subject_isolation() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let api = test_api(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("delete me").unwrap(),
|
||||
"private behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let other = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("keep me").unwrap(),
|
||||
"unchanged".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-owner").await;
|
||||
let stranger = seed_test_api_token(api.store.as_ref(), "subject-delete-stranger");
|
||||
for (token, workspace, expected) in [
|
||||
(None, TEST_WORKSPACE_ID, StatusCode::UNAUTHORIZED),
|
||||
(
|
||||
Some(stranger.as_str()),
|
||||
TEST_WORKSPACE_ID,
|
||||
StatusCode::FORBIDDEN,
|
||||
),
|
||||
(
|
||||
Some(owner.as_str()),
|
||||
"another-workspace",
|
||||
StatusCode::NOT_FOUND,
|
||||
),
|
||||
] {
|
||||
let (status, body) =
|
||||
delete_request(&api, &subject.id, "same-operation", token, workspace).await;
|
||||
assert_eq!(status, expected, "{body}");
|
||||
assert!(store.subject(&subject.id).unwrap().is_some());
|
||||
}
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"same-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||
assert_eq!(
|
||||
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"unchanged"
|
||||
);
|
||||
assert_eq!(store.list_subjects(100).unwrap().items.len(), 1);
|
||||
let (status, replay) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"same-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{replay}");
|
||||
assert_eq!(replay, body);
|
||||
let (status, _) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"different-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
let (status, _) = delete_request(
|
||||
&api,
|
||||
&other.id,
|
||||
"same-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
assert!(store.subject(&other.id).unwrap().is_some());
|
||||
let fresh = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("fresh").unwrap())
|
||||
.unwrap();
|
||||
assert_ne!(fresh.id, subject.id);
|
||||
assert!(store.subject_permissions(&fresh.id).unwrap().is_none());
|
||||
assert_eq!(
|
||||
store
|
||||
.pending_staging_candidates(&fresh.id, 100)
|
||||
.unwrap()
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_inactive_subject_is_preserved_until_explicit_delete_and_retire_route_is_gone() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let api = test_api(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("legacy fixture").unwrap(),
|
||||
"retained original".into(),
|
||||
)
|
||||
.unwrap();
|
||||
api.feature_storage.open(&api.subjektiv_registration).unwrap()
|
||||
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute("UPDATE subjects SET state='retired',record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&subject.id])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert_eq!(
|
||||
store.subject(&subject.id).unwrap().unwrap().state,
|
||||
crate::subjektiv::SubjectState::LegacyRetired
|
||||
);
|
||||
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-legacy").await;
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!(
|
||||
"/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/retire",
|
||||
subject.id
|
||||
))
|
||||
.header("authorization", format!("Bearer {owner}"))
|
||||
.body(Body::empty())
|
||||
.unwrap();
|
||||
let old = build_router(api.clone()).oneshot(request).await.unwrap();
|
||||
assert_eq!(old.status(), StatusCode::METHOD_NOT_ALLOWED);
|
||||
assert_eq!(
|
||||
store.subject(&subject.id).unwrap().unwrap().behavior_md,
|
||||
"retained original"
|
||||
);
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"explicit-legacy-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn unconfirmed_worker_shutdown_is_not_completion_and_exact_retry_converges() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("delete connected body").unwrap(),
|
||||
"private behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let Json(created) = create_workspace_worker(
|
||||
State(api.clone()),
|
||||
HeaderMap::new(),
|
||||
Json(CreateWorkspaceWorkerRequest {
|
||||
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(),
|
||||
display_name: "Subject deletion boundary".into(),
|
||||
singleton_key: None,
|
||||
profile: Some("builtin:companion".into()),
|
||||
ticket_assignment: None,
|
||||
initial_submit: vec![],
|
||||
workdir_attachments: vec![],
|
||||
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
|
||||
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest {
|
||||
subject_id: subject.id.clone(),
|
||||
}),
|
||||
},
|
||||
control_operation_id: None,
|
||||
execution_grant: None,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id);
|
||||
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unknown").await;
|
||||
*execution.subject_delete_failure.lock().unwrap() = Some("shutdown completion unknown".into());
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"unknown-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], false);
|
||||
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||
assert!(
|
||||
!store
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||
api.config_store.with_conn(|conn| {
|
||||
conn.execute_batch("CREATE TRIGGER injected_worker_catalog_failure BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected catalog failure'); END;")?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
let (status, midpoint) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"unknown-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{midpoint}");
|
||||
assert_eq!(midpoint["completed"], false);
|
||||
assert!(matches!(
|
||||
api.runtime.worker(&worker),
|
||||
Err(RuntimeRegistryError::UnknownWorker { .. })
|
||||
));
|
||||
assert!(
|
||||
api.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
api.config_store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER injected_worker_catalog_failure;")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"unknown-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true, "{body}");
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let removal = api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
removal.plan.state,
|
||||
crate::retention::WorkerRemovalPlanState::Succeeded
|
||||
);
|
||||
assert_eq!(
|
||||
removal.runtime_request.session_disposition,
|
||||
worker_runtime::retention::SessionDisposition::Archive
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn captured_subject_job_result_cannot_deliver_after_domain_fence() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("notification fence").unwrap())
|
||||
.unwrap();
|
||||
let target = spawn_ticket_check_source(&api, "subject-notification-target");
|
||||
let worker = spawn_ticket_check_source(&api, "subject-notification-result");
|
||||
let request = BackendJobRequest {
|
||||
job_id: "captured-subject-notification".into(),
|
||||
purpose: "subject_notification_contract".into(),
|
||||
input_ref: "subject:notification".into(),
|
||||
input: json!({"private":"input"}),
|
||||
instruction: "Read input".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: crate::backend_job::BackendJobGrants {
|
||||
subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant {
|
||||
subject_id: subject.id.clone(),
|
||||
candidate_ids: vec!["candidate".into()],
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
serialization_key: None,
|
||||
source_worker: Some(target.clone()),
|
||||
notification_target: Some(target),
|
||||
limits: Default::default(),
|
||||
};
|
||||
// Enter at the committed result boundary: execution/result validation is
|
||||
// outside this notification test, but routing and Runtime transport are real.
|
||||
let now = Utc::now().to_rfc3339();
|
||||
let reserved = api
|
||||
.store
|
||||
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &now)
|
||||
.unwrap();
|
||||
let attempt = &reserved.attempt.attempt_id;
|
||||
api.store
|
||||
.claim_backend_job_attempt_dispatch(TEST_WORKSPACE_ID, &request.job_id, attempt, &now)
|
||||
.unwrap();
|
||||
api.store
|
||||
.bind_backend_job_attempt_worker(
|
||||
TEST_WORKSPACE_ID,
|
||||
&request.job_id,
|
||||
attempt,
|
||||
&worker,
|
||||
Some("run"),
|
||||
&now,
|
||||
)
|
||||
.unwrap();
|
||||
let captured = api
|
||||
.store
|
||||
.accept_backend_job_result(
|
||||
TEST_WORKSPACE_ID,
|
||||
&worker,
|
||||
&BackendJobResultSubmission {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id: attempt.clone(),
|
||||
input_digest: request.input_digest().unwrap(),
|
||||
result: json!({"private":"result"}),
|
||||
},
|
||||
&now,
|
||||
)
|
||||
.unwrap();
|
||||
let delivery = crate::backend_job::delivery_id(&request.job_id, attempt);
|
||||
assert!(
|
||||
api.store
|
||||
.job_notification_source(TEST_WORKSPACE_ID, &request.job_id, attempt)
|
||||
.is_ok()
|
||||
);
|
||||
execution.take_inputs();
|
||||
store
|
||||
.begin_subject_deletion(&subject.id, "notification-delete")
|
||||
.unwrap();
|
||||
notification_delivery::deliver_job_result(&api, &captured);
|
||||
notification_delivery::recover_job_notifications(&api);
|
||||
assert!(
|
||||
execution.take_inputs().is_empty(),
|
||||
"captured or recovered result must not escape the domain fence"
|
||||
);
|
||||
assert!(
|
||||
api.store
|
||||
.job_notification_request(TEST_WORKSPACE_ID, &delivery)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn domain_purge_failure_keeps_fence_and_backend_reopen_resumes_exact_operation() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let api = test_api(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("retry purge").unwrap(),
|
||||
"do not resurrect".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let other = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("untouched").unwrap(),
|
||||
"keep".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let database = api
|
||||
.feature_storage
|
||||
.open(&api.subjektiv_registration)
|
||||
.unwrap();
|
||||
database.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute_batch("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected domain purge failure'); END;")?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
let owner =
|
||||
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-purge-failure").await;
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"purge-retry",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert!(status.is_server_error(), "{status}: {body}");
|
||||
assert!(
|
||||
!store
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||
assert_eq!(
|
||||
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"keep"
|
||||
);
|
||||
database
|
||||
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute_batch("DROP TRIGGER injected_subject_purge_failure;")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let mut reopened = api.clone();
|
||||
let config_store =
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap());
|
||||
reopened.config_store = config_store.clone();
|
||||
reopened.store = config_store;
|
||||
let recovery_api = reopened.clone();
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
reopened
|
||||
.host_executors
|
||||
.host()
|
||||
.unwrap()
|
||||
.run(move || handle.block_on(subjektiv_deletion::recover(&recovery_api)))
|
||||
.await
|
||||
.unwrap();
|
||||
let recovered = open_subjektiv_store(&reopened)
|
||||
.unwrap()
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(recovered.operation_id, "purge-retry");
|
||||
assert!(recovered.completed);
|
||||
let (status, body) = delete_request(
|
||||
&reopened,
|
||||
&subject.id,
|
||||
"purge-retry",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert_eq!(
|
||||
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"keep"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn pending_consolidation_is_purged_and_captured_dispatch_and_late_result_are_rejected() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("delete running Job").unwrap())
|
||||
.unwrap();
|
||||
let candidate: memory::extract::StagingRecord = serde_json::from_value(json!({
|
||||
"schema_version": 2, "id": "delete-job-candidate", "extract_run_id": "extract",
|
||||
"source": {"segment_id":"segment", "range":[1,2]}, "kind":"constraint",
|
||||
"claim":"private candidate marker", "why_useful":"retained only until explicit erasure",
|
||||
"evidence": [], "source_refs": []
|
||||
}))
|
||||
.unwrap();
|
||||
store
|
||||
.stage_candidate(crate::subjektiv::SubjectStagingRecord::attach(
|
||||
&subject.id,
|
||||
candidate,
|
||||
))
|
||||
.unwrap();
|
||||
let started = start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true },
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(started.status, "started");
|
||||
let job = api
|
||||
.store
|
||||
.find_active_backend_job_for_resource(
|
||||
TEST_WORKSPACE_ID,
|
||||
&crate::backend_job::subjektiv_serialization_key(&subject.id),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let attempt = api
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&job.request.job_id,
|
||||
&crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let worker = attempt.worker.clone().unwrap();
|
||||
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||
let captured = BackendJobReservation {
|
||||
job: job.clone(),
|
||||
attempt: attempt.clone(),
|
||||
replayed: true,
|
||||
resource_reused: false,
|
||||
};
|
||||
let owner =
|
||||
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-consolidation").await;
|
||||
*execution.subject_delete_failure.lock().unwrap() = Some("completion unknown".into());
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"delete-job",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], false);
|
||||
assert!(
|
||||
api.dispatch_backend_job_reservation(captured.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
api.accept_backend_job_result(
|
||||
&worker,
|
||||
&BackendJobResultSubmission {
|
||||
job_id: job.request.job_id.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
input_digest: job.input_digest.clone(),
|
||||
result: json!({"late":true}),
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"delete-job",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert!(
|
||||
api.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
api.store
|
||||
.get_backend_job_attempt(TEST_WORKSPACE_ID, &job.request.job_id, &attempt.attempt_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(api.dispatch_backend_job_reservation(captured).is_err());
|
||||
assert!(api.dispatch_backend_job(&job.request).is_err());
|
||||
let removal = api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
removal.plan.state,
|
||||
crate::retention::WorkerRemovalPlanState::Succeeded
|
||||
);
|
||||
assert_eq!(
|
||||
removal.runtime_request.session_disposition,
|
||||
worker_runtime::retention::SessionDisposition::Purge
|
||||
);
|
||||
assert_eq!(
|
||||
removal.runtime_request.diagnostics_disposition,
|
||||
worker_runtime::retention::DiagnosticsDisposition::Purge
|
||||
);
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(".test-embedded-runtime-store/workers")
|
||||
.join(&worker.worker_id)
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unmaterialized_body_and_job_reservations_finish_only_with_runtime_storage_absence_proof() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, _) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let owner =
|
||||
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unmaterialized").await;
|
||||
for is_job in [false, true] {
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("reserved before deletion").unwrap())
|
||||
.unwrap();
|
||||
let singleton = (!is_job).then(|| format!("subjektiv:{}", subject.id));
|
||||
let allocation = if is_job {
|
||||
let request = BackendJobRequest {
|
||||
job_id: "unmaterialized-subject-job".into(),
|
||||
purpose: "subject_reservation_contract".into(),
|
||||
input_ref: "subject:reserved".into(),
|
||||
input: json!({"private":"reserved input"}),
|
||||
instruction: "read".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: crate::backend_job::BackendJobGrants {
|
||||
subjektiv_consolidation: Some(
|
||||
crate::backend_job::SubjektivConsolidationGrant {
|
||||
subject_id: subject.id.clone(),
|
||||
candidate_ids: vec!["candidate".into()],
|
||||
},
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: Default::default(),
|
||||
};
|
||||
let reserved = api
|
||||
.store
|
||||
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339())
|
||||
.unwrap();
|
||||
crate::backend_job::allocation_key(&reserved.attempt.attempt_id)
|
||||
} else {
|
||||
"unmaterialized-subject-body".into()
|
||||
};
|
||||
let settings = api
|
||||
.config_store
|
||||
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
|
||||
.unwrap();
|
||||
let reserved = api
|
||||
.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
&allocation,
|
||||
"unmaterialized-fingerprint",
|
||||
singleton.as_deref(),
|
||||
&settings,
|
||||
)
|
||||
.unwrap();
|
||||
let worker =
|
||||
RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, reserved.worker_id.to_string());
|
||||
assert!(matches!(
|
||||
api.runtime.worker(&worker),
|
||||
Err(RuntimeRegistryError::UnknownWorker { .. })
|
||||
));
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
&format!("delete-reserved-{is_job}"),
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
|
||||
let proved = api.config_store.with_conn(|conn| Ok(conn.query_row("SELECT originals_absent FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", rusqlite::params![TEST_WORKSPACE_ID,subject.id,worker.runtime_id,worker.worker_id], |row|row.get::<_,bool>(0))?)).unwrap();
|
||||
assert!(proved);
|
||||
assert!(
|
||||
api.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.is_none(),
|
||||
"absence is not a fabricated Purge/archive receipt"
|
||||
);
|
||||
assert!(
|
||||
api.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
&allocation,
|
||||
"unmaterialized-fingerprint",
|
||||
singleton.as_deref(),
|
||||
&settings
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
if let Some(singleton) = singleton {
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1143,6 +1143,8 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
||||
&Utc::now().to_rfc3339(),
|
||||
)
|
||||
.unwrap();
|
||||
// Discard only the original delivery, before recovery can automatically retry.
|
||||
f.execution.take_inputs();
|
||||
f.api.recover_backend_job_worker_cleanups().unwrap();
|
||||
wait_for_backend_job_worker_cleanup(
|
||||
&f.api,
|
||||
@@ -1151,23 +1153,64 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
||||
BackendJobWorkerCleanupState::Completed,
|
||||
)
|
||||
.await;
|
||||
// Cleanup commits before domain reconciliation; wait for its authoritative outcome.
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
// Cleanup commits before domain reconciliation. Job Failed is only transient:
|
||||
// reconcile_unknown fences this attempt, then automatically reserves/queues its
|
||||
// retry. Observe the immutable attempt outcome and the delivered new generation,
|
||||
// not the Job's transient state; a manual retry would race the automatic one.
|
||||
let reconciliation = tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
loop {
|
||||
let job = f
|
||||
.api
|
||||
.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
if job.state == BackendJobState::Failed {
|
||||
break;
|
||||
let original = f.api.store.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&dispatched.attempt.attempt_id,
|
||||
).unwrap().unwrap();
|
||||
if original.state == BackendJobAttemptState::Failed {
|
||||
assert_eq!(original.failure_category.as_deref(), Some("journal_attempt_fenced"));
|
||||
assert_eq!(original.worker_cleanup_state, Some(BackendJobWorkerCleanupState::Completed));
|
||||
assert!(original.worker_cleanup_completed_at.is_some());
|
||||
if let Some(generation) = f.store.active_journal_generation(&f.host, &f.id).unwrap()
|
||||
&& generation.job.attempt_id != original.attempt_id
|
||||
&& let Some(retry) = f.api.store.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&generation.job.attempt_id,
|
||||
).unwrap()
|
||||
&& retry.state == BackendJobAttemptState::Dispatched
|
||||
&& let Some(resumed) = &retry.worker
|
||||
// Worker binding precedes input delivery/attestation. ReadState
|
||||
// succeeds only once the new generation's delivery is attested.
|
||||
&& subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()
|
||||
{
|
||||
break (original, retry, generation);
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
assert!(
|
||||
reconciliation.is_ok(),
|
||||
"post-cleanup reconciliation timed out: job={:?}, original_attempt={:?}, active_generation={:?}",
|
||||
f.api
|
||||
.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||
.unwrap(),
|
||||
f.api
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&dispatched.attempt.attempt_id
|
||||
)
|
||||
.unwrap(),
|
||||
f.store.active_journal_generation(&f.host, &f.id).unwrap(),
|
||||
);
|
||||
let (fenced, retry, generation) = reconciliation.unwrap();
|
||||
assert_eq!(fenced.worker, dispatched.attempt.worker);
|
||||
assert_eq!(retry.attempt, dispatched.attempt.attempt + 1);
|
||||
assert_eq!(generation.job.job_id, f.request.job_id);
|
||||
assert_eq!(generation.job.attempt_id, retry.attempt_id);
|
||||
assert_eq!(retry.input_digest, f.request.input_digest().unwrap());
|
||||
f.assert_purged(worker);
|
||||
assert!(f.store.journal_retention_required(&f.host).unwrap());
|
||||
assert_eq!(
|
||||
@@ -1187,25 +1230,25 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
||||
1,
|
||||
"source pin survives failed/partial Job cleanup"
|
||||
);
|
||||
f.execution.take_inputs();
|
||||
let retry = f
|
||||
let job = f
|
||||
.api
|
||||
.store
|
||||
.reserve_backend_job_retry(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&Utc::now().to_rfc3339(),
|
||||
)
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(retry.job.request.input, f.request.input);
|
||||
assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap());
|
||||
let retry = f.api.dispatch_backend_job_reservation(retry).unwrap();
|
||||
let resumed = retry.attempt.worker.as_ref().unwrap();
|
||||
assert_eq!(job.state, BackendJobState::Pending);
|
||||
assert_eq!(job.current_attempt, retry.attempt);
|
||||
assert_eq!(
|
||||
job.load_executable_request().unwrap().input,
|
||||
f.request.input
|
||||
);
|
||||
assert_eq!(job.input_digest, f.request.input_digest().unwrap());
|
||||
let resumed = retry.worker.as_ref().unwrap();
|
||||
let delivered = f.execution.take_inputs();
|
||||
assert_eq!(delivered.len(), 1);
|
||||
assert_eq!(
|
||||
delivered[0].1,
|
||||
f.request.worker_input(&retry.attempt.attempt_id).unwrap()
|
||||
f.request.worker_input(&retry.attempt_id).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
f.store
|
||||
|
||||
@@ -92,6 +92,13 @@ use crate::backend_job::{
|
||||
use crate::workspace_deletion::WorkspaceDeletionStore;
|
||||
use crate::{Error, Result};
|
||||
|
||||
#[path = "subject_deletion_store.rs"]
|
||||
mod subject_deletion_store;
|
||||
pub(crate) use subject_deletion_store::{
|
||||
prior_removal_matches as subject_deletion_prior_removal_matches,
|
||||
worker_kind as subject_deletion_worker_kind,
|
||||
};
|
||||
|
||||
#[path = "activation.rs"]
|
||||
mod activation;
|
||||
pub use activation::ActivationCodeIssue;
|
||||
@@ -119,7 +126,7 @@ pub use ticket::{
|
||||
};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 104;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 105;
|
||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
@@ -447,6 +454,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: "Converge Passkey results and Job input retention",
|
||||
apply: migrate_schema_103_variants_to_v104,
|
||||
},
|
||||
Migration {
|
||||
version: 105,
|
||||
name: "Durable Subject deletion snapshots and Job admission fences",
|
||||
apply: subject_deletion_store::migrate,
|
||||
},
|
||||
];
|
||||
|
||||
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
||||
@@ -3000,6 +3012,7 @@ impl SqliteWorkspaceStore {
|
||||
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
subject_deletion_store::require_worker_admission(&tx, workspace_id, allocation_key, singleton_key)?;
|
||||
let fixed_worker_id = prepare(&tx)?;
|
||||
let existing = tx
|
||||
.query_row(
|
||||
@@ -7455,6 +7468,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let deadline = backend_job_deadline(now, request.limits.timeout_seconds)?;
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, request)?;
|
||||
if let Some(job) = read_backend_job(&tx, workspace_id, &request.job_id)? {
|
||||
if job.intent_fingerprint != fingerprint {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
@@ -7532,6 +7546,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
return Ok(job);
|
||||
}
|
||||
if reason == JobInputRetirementReason::Completed {
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
|
||||
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
|
||||
@@ -7562,6 +7577,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let mut job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let previous_id = attempt_id(job_id, job.current_attempt);
|
||||
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
|
||||
@@ -7701,6 +7717,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||
@@ -7764,6 +7781,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||
@@ -7827,6 +7845,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| {
|
||||
@@ -7880,6 +7899,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{}`", submission.job_id)))?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
|
||||
if submission.input_digest != job.input_digest
|
||||
|| attempt.input_digest != submission.input_digest
|
||||
@@ -7908,6 +7928,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
tx.commit()?;
|
||||
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
|
||||
}
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
|
||||
return Err(Error::InvalidInput(
|
||||
@@ -7972,6 +7993,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
|
||||
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
|
||||
if &attempt != expected || job.current_attempt != attempt.attempt
|
||||
@@ -8277,6 +8299,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
) -> Result<(BackendJobDeliveryRecord, bool)> {
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let deleted:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id=?1 AND job_id=?2)",params![workspace_id,job_id],|r|r.get(0))?;
|
||||
if deleted {return Err(Error::RepositoryConflict("subject_deleted: Job delivery fenced".into()));}
|
||||
if let Some(existing) = read_backend_job_delivery(&tx, workspace_id, delivery_id)? {
|
||||
if existing.job_id != job_id || existing.target != *target {
|
||||
return Err(Error::InvalidInput(
|
||||
@@ -15545,6 +15569,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
||||
notification_routing::jobs::install(conn)?;
|
||||
notification_routing::install_business_events(conn)?;
|
||||
notification_routing::upgrade_result_reports(conn)?;
|
||||
subject_deletion_store::install_notification_erasure(conn)?;
|
||||
ticket::verify_sqlite_ticket_schema(conn).map_err(|error| {
|
||||
Error::Store(format!(
|
||||
"fresh Workspace Ticket schema verification failed: {error}"
|
||||
@@ -15583,10 +15608,17 @@ fn create_workspace_schema_structure_from_sql(
|
||||
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
||||
|
||||
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
|
||||
conn.execute_batch(&schema_sql.replace(
|
||||
conn.execute_batch(
|
||||
&schema_sql
|
||||
.replace(
|
||||
"-- Ticket-owned assignment schema is composed by store.rs.",
|
||||
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
|
||||
))?;
|
||||
)
|
||||
.replace(
|
||||
"-- Subject deletion schema is composed by store.rs.",
|
||||
include_str!("subject_deletion_schema.sql"),
|
||||
),
|
||||
)?;
|
||||
let composed_sql = r#"
|
||||
ALTER TABLE typed_tickets RENAME TO typed_tickets_legacy_v66;
|
||||
CREATE TABLE typed_tickets (
|
||||
@@ -15809,6 +15841,9 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? >= 96 {
|
||||
notification_routing::jobs::verify(conn)?;
|
||||
}
|
||||
if current_schema_version(conn)? >= 105 {
|
||||
subject_deletion_store::verify(conn)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -16223,6 +16258,8 @@ fn current_worker_singleton_owner(
|
||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
||||
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \
|
||||
THEN 'created' ELSE reservation.state END \
|
||||
FROM worker_singleton_owners owner \
|
||||
LEFT JOIN worker_create_reservations reservation \
|
||||
@@ -19265,6 +19302,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
||||
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
|
||||
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
|
||||
WorkspaceSchemaMigrationStep {version:105, name:"Durable Subject deletion snapshots and Job admission fences".into()},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -19379,6 +19417,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(102, "Durable failed Subject creation recovery".to_string()),
|
||||
(103, "Passkey metadata and atomic addition results".to_string()),
|
||||
(104, "Converge Passkey results and Job input retention".to_string()),
|
||||
(105, "Durable Subject deletion snapshots and Job admission fences".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -22664,7 +22703,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(101, "Durable explicit Subject body release".into()),
|
||||
(102, "Durable failed Subject creation recovery".into()),
|
||||
(103, "Passkey metadata and atomic addition results".into()),
|
||||
(104, "Converge Passkey results and Job input retention".into())]);
|
||||
(104, "Converge Passkey results and Job input retention".into()),
|
||||
(105, "Durable Subject deletion snapshots and Job admission fences".into())]);
|
||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||
}
|
||||
@@ -26857,12 +26897,19 @@ INSERT INTO worker_registry (
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
store.with_conn(|conn| {
|
||||
// This fixture deliberately reconstructs schema 102 from fresh.
|
||||
// Remove the newer Subject fence composition, not just its receipt.
|
||||
let triggers=conn.prepare("SELECT name,tbl_name FROM sqlite_schema WHERE type='trigger' AND tbl_name LIKE 'notification_%' AND name NOT LIKE 'subject_deleted_%' AND sql LIKE '%subject_deletion_jobs%'")?
|
||||
.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;}
|
||||
for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;}
|
||||
conn.execute_batch("DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?;
|
||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
DROP TRIGGER passkey_metadata_insert;
|
||||
DROP TABLE passkey_addition_receipts;
|
||||
DROP TABLE passkey_metadata;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=105;
|
||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
||||
}).unwrap();
|
||||
drop(store);
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
-- Content-free, monotonic main-DB fence. The Host installs the domain fence first
|
||||
-- and holds its Subject gate across snapshot/admission. No Subject contents live here.
|
||||
CREATE TABLE subject_deletions (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
operation_id TEXT NOT NULL CHECK(length(operation_id)>0),
|
||||
completed INTEGER NOT NULL DEFAULT 0 CHECK(completed IN (0,1)),
|
||||
jobs_erased INTEGER NOT NULL DEFAULT 0 CHECK(jobs_erased IN (0,1)),
|
||||
PRIMARY KEY(workspace_id,subject_id),
|
||||
UNIQUE(workspace_id,operation_id),
|
||||
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE subject_deletion_workers (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
is_job INTEGER NOT NULL CHECK(is_job IN (0,1)),
|
||||
-- Exact canonical success already present at prepare, never a later receipt.
|
||||
prior_removal_operation_id TEXT REFERENCES worker_removal_operations(operation_id),
|
||||
-- Confirmed by the scoped Runtime storage owner, not catalog absence or Stop ACK.
|
||||
originals_absent INTEGER NOT NULL DEFAULT 0 CHECK(originals_absent IN (0,1)),
|
||||
PRIMARY KEY(workspace_id,subject_id,runtime_id,worker_id),
|
||||
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TRIGGER subject_deletion_worker_prior_insert BEFORE INSERT ON subject_deletion_workers
|
||||
WHEN NEW.prior_removal_operation_id IS NOT NULL AND NOT EXISTS(
|
||||
SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=NEW.prior_removal_operation_id
|
||||
AND r.workspace_id=NEW.workspace_id AND r.runtime_id=NEW.runtime_id
|
||||
AND r.worker_id=NEW.worker_id AND r.state='succeeded')
|
||||
BEGIN SELECT RAISE(ABORT,'Subject deletion prior removal must be an exact canonical success'); END;
|
||||
CREATE TRIGGER subject_deletion_worker_identity_immutable BEFORE UPDATE OF
|
||||
workspace_id,subject_id,runtime_id,worker_id,prior_removal_operation_id ON subject_deletion_workers
|
||||
BEGIN SELECT RAISE(ABORT,'Subject deletion Worker snapshot identity is immutable'); END;
|
||||
CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers
|
||||
WHEN OLD.originals_absent=1 AND NEW.originals_absent=0
|
||||
BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END;
|
||||
-- Job/attempt identities remain after payload erasure, preventing stale allocation
|
||||
-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs.
|
||||
CREATE TABLE subject_deletion_jobs (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
PRIMARY KEY(workspace_id,job_id),
|
||||
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE subject_deletion_allocations (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
allocation_key TEXT NOT NULL,
|
||||
PRIMARY KEY(workspace_id,allocation_key),
|
||||
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TRIGGER subject_deleted_job_insert BEFORE INSERT ON backend_jobs
|
||||
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
|
||||
(d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_journal.subject_id') OR
|
||||
d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_consolidation.subject_id')))
|
||||
OR EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job admission fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_job_result BEFORE UPDATE OF state,result_json ON backend_jobs
|
||||
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job result fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_attempt_insert BEFORE INSERT ON backend_job_attempts
|
||||
WHEN EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job retry fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_attempt_result BEFORE UPDATE OF state,result_json,runtime_id,worker_id,runtime_run_id ON backend_job_attempts
|
||||
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job attempt fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_worker_reservation BEFORE INSERT ON worker_create_reservations
|
||||
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
|
||||
NEW.singleton_key IN ('subjektiv:'||d.subject_id,'subjektiv-consolidation:'||d.subject_id))
|
||||
OR EXISTS(SELECT 1 FROM subject_deletion_allocations d WHERE d.workspace_id=NEW.workspace_id AND d.allocation_key=NEW.allocation_key)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Worker allocation fenced'); END;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -94,6 +94,12 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"workdir_registry",
|
||||
"workdir_removal_operations",
|
||||
"worker_control_grants",
|
||||
// Snapshot children precede their fence parent and canonical Worker removal
|
||||
// receipts (subject_deletion_workers.prior_removal_operation_id).
|
||||
"subject_deletion_allocations",
|
||||
"subject_deletion_jobs",
|
||||
"subject_deletion_workers",
|
||||
"subject_deletions",
|
||||
"subject_worker_releases",
|
||||
"subject_create_recoveries",
|
||||
"worker_create_reservations",
|
||||
@@ -1276,6 +1282,134 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() {
|
||||
let (store, owner, workspace_id) = setup();
|
||||
let tables = [
|
||||
"subject_deletion_allocations",
|
||||
"subject_deletion_jobs",
|
||||
"subject_deletion_workers",
|
||||
"subject_deletions",
|
||||
"worker_removal_operations",
|
||||
];
|
||||
// Check the explicit FK ordering as well as execution (the purge defers FKs).
|
||||
let position = |table| {
|
||||
WORKSPACE_DELETION_PURGE_TABLES
|
||||
.iter()
|
||||
.position(|name| *name == table)
|
||||
.unwrap()
|
||||
};
|
||||
for child in &tables[..3] {
|
||||
assert!(position(*child) < position("subject_deletions"));
|
||||
}
|
||||
assert!(position("subject_deletion_workers") < position("worker_removal_operations"));
|
||||
for workspace in ["workspace-a", "workspace-b"] {
|
||||
store.with_conn(|conn| {
|
||||
conn.execute("INSERT INTO worker_removal_operations(
|
||||
operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id,
|
||||
worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,
|
||||
archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at)
|
||||
VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest',
|
||||
'purge','purge','forever','purge','[]','succeeded','removed','1','1')", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased)
|
||||
VALUES(?1,'subject','subject-delete',1,1)", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent)
|
||||
VALUES(?1,'subject','runtime','worker',1,?1,1)", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
let rows = |workspace: &str| {
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
tables
|
||||
.iter()
|
||||
.map(|table| {
|
||||
let mut statement = conn
|
||||
.prepare(&format!("SELECT * FROM {table} WHERE workspace_id=?1"))?;
|
||||
let count = statement.column_count();
|
||||
Ok(statement
|
||||
.query_map([workspace], |row| {
|
||||
(0..count)
|
||||
.map(|index| row.get::<_, rusqlite::types::Value>(index))
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()
|
||||
})
|
||||
.unwrap()
|
||||
};
|
||||
let before = rows("workspace-a");
|
||||
let other = rows("workspace-b");
|
||||
assert!(before.iter().all(|table| table.len() == 1));
|
||||
let preflight = store
|
||||
.workspace_deletion_preflight(&owner, &workspace_id)
|
||||
.unwrap();
|
||||
let operation = store
|
||||
.reserve_workspace_deletion(
|
||||
&owner,
|
||||
&workspace_id,
|
||||
&WorkspaceDeletionRequest {
|
||||
operation_id: "purge-subject-snapshots".into(),
|
||||
expected_workspace_updated_at: preflight.expected_workspace_updated_at,
|
||||
confirmation: "Alpha".into(),
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.operation;
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch(
|
||||
"CREATE TRIGGER reject_subject_snapshot_purge BEFORE DELETE ON workspaces
|
||||
BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
rows("workspace-a"),
|
||||
before,
|
||||
"rollback restores snapshots and canonical receipt"
|
||||
);
|
||||
assert_eq!(rows("workspace-b"), other);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER reject_subject_snapshot_purge")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let completed = store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, WorkspaceDeletionState::Succeeded);
|
||||
assert!(rows("workspace-a").iter().all(|table| table.is_empty()));
|
||||
assert_eq!(rows("workspace-b"), other);
|
||||
assert_eq!(
|
||||
store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
.unwrap(),
|
||||
completed
|
||||
);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(
|
||||
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
|
||||
row.get::<_, i64>(0)
|
||||
})?,
|
||||
0
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_is_idempotent_and_removes_workspace_scoped_rows() {
|
||||
let (store, owner, workspace_id) = setup();
|
||||
|
||||
+166
-111
@@ -11127,6 +11127,40 @@
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"SubjektivSubjectDeleteRequest": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"operation_id": {
|
||||
"description": "Stable identity retained for every retry, including unknown outcomes.",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation_id"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"SubjektivSubjectDeleteResponse": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"completed": {
|
||||
"description": "False confirms the deletion fence, not external stop or erasure completion.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"operation_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"subject_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"subject_id",
|
||||
"operation_id",
|
||||
"completed"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"SubjektivSubjectListQuery": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -11228,12 +11262,20 @@
|
||||
"type": "object"
|
||||
},
|
||||
"SubjektivSubjectState": {
|
||||
"oneOf": [
|
||||
{
|
||||
"enum": [
|
||||
"active",
|
||||
"retired"
|
||||
"active"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"const": "retired",
|
||||
"description": "Saved compatibility only; not a current lifecycle operation.",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
"SubjektivWorkerConnectionRequest": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -17129,6 +17171,10 @@
|
||||
"additionalProperties": false,
|
||||
"description": "Backend-authoritative permissions for the current Workspace actor.",
|
||||
"properties": {
|
||||
"delete_subjects": {
|
||||
"description": "Explicit Subject deletion authority for the current authenticated actor.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"delete_workspace": {
|
||||
"type": "boolean"
|
||||
},
|
||||
@@ -17146,7 +17192,8 @@
|
||||
"manage_repositories",
|
||||
"manage_secrets",
|
||||
"manage_runtimes",
|
||||
"delete_workspace"
|
||||
"delete_workspace",
|
||||
"delete_subjects"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
@@ -18302,7 +18349,7 @@
|
||||
"info": {
|
||||
"title": "Yoi Server API",
|
||||
"version": "0.1.0",
|
||||
"x-yoi-source-digest": "sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4"
|
||||
"x-yoi-source-digest": "sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e"
|
||||
},
|
||||
"openapi": "3.1.0",
|
||||
"paths": {
|
||||
@@ -35014,6 +35061,121 @@
|
||||
}
|
||||
},
|
||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}": {
|
||||
"delete": {
|
||||
"operationId": "subjektiv_subject_delete",
|
||||
"parameters": [
|
||||
{
|
||||
"explode": false,
|
||||
"in": "path",
|
||||
"name": "workspace_id",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/string"
|
||||
},
|
||||
"style": "simple"
|
||||
},
|
||||
{
|
||||
"explode": false,
|
||||
"in": "path",
|
||||
"name": "subject_id",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/string"
|
||||
},
|
||||
"style": "simple"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/SubjektivSubjectDeleteRequest"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/SubjektivSubjectDeleteResponse"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful response"
|
||||
},
|
||||
"400": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"401": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"403": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"404": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"409": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"500": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"bearerAuth": []
|
||||
},
|
||||
{
|
||||
"browserSession": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"get": {
|
||||
"operationId": "subjektiv_subject_get",
|
||||
"parameters": [
|
||||
@@ -35790,113 +35952,6 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire": {
|
||||
"post": {
|
||||
"operationId": "subjektiv_subject_retire",
|
||||
"parameters": [
|
||||
{
|
||||
"explode": false,
|
||||
"in": "path",
|
||||
"name": "workspace_id",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/string"
|
||||
},
|
||||
"style": "simple"
|
||||
},
|
||||
{
|
||||
"explode": false,
|
||||
"in": "path",
|
||||
"name": "subject_id",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/string"
|
||||
},
|
||||
"style": "simple"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/SubjektivSubjectResponse"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful response"
|
||||
},
|
||||
"400": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"401": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"403": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"404": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"409": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
},
|
||||
"500": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/RepositoryApiError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Error response"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"bearerAuth": []
|
||||
},
|
||||
{
|
||||
"browserSession": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/surface": {
|
||||
"get": {
|
||||
"operationId": "subjektiv_resident_surface",
|
||||
|
||||
@@ -32,6 +32,14 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path": "crates/subjektiv/src/deletion/tests.rs",
|
||||
"kind": "saved_data",
|
||||
"reason": "schema 6の固定保存Subject行を投入し、最新migrationが旧retiredとarchive原文を黙って消去・再活性化せず、明示削除だけで消去する検証に必要な旧SQL列名。現行writerや競合制御には使わない。",
|
||||
"lines": [
|
||||
{ "sha256": "defca5951b3d48785514b25912044fa3fc252659311b09a31fa2e64af2a19049", "count": 1 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"path": "crates/subjektiv/src/journal/memory_links/tests.rs",
|
||||
"kind": "saved_data",
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "subject-delete-dark",
|
||||
"title": "Subject deletion confirmation and retry",
|
||||
"baseUrl": "http://127.0.0.1:15287",
|
||||
"colorScheme": "dark",
|
||||
"reducedMotion": "reduce",
|
||||
"personas": [
|
||||
{
|
||||
"id": "fixture",
|
||||
"label": "Authenticated owner (synthetic HTTP)",
|
||||
"auth": {
|
||||
"kind": "anonymous"
|
||||
}
|
||||
}
|
||||
],
|
||||
"viewports": [
|
||||
{
|
||||
"label": "desktop",
|
||||
"width": 1440,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"label": "narrow",
|
||||
"width": 768,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"label": "mobile",
|
||||
"width": 390,
|
||||
"height": 844
|
||||
}
|
||||
],
|
||||
"routes": [
|
||||
{
|
||||
"id": "subject",
|
||||
"label": "Subject detail",
|
||||
"path": "/w/memory-review/memory/pending-subject",
|
||||
"goal": "Review the Subject before irreversible deletion.",
|
||||
"dataState": "Active Subject, no current Worker; owner permissions from Backend projection.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": ".subject-overview"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Subject detail"
|
||||
},
|
||||
{
|
||||
"id": "confirmation",
|
||||
"label": "Irreversible deletion scope and confirmation",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-delete-open"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=confirming]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "pending",
|
||||
"label": "Fence saved; erasure unconfirmed",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".delete-confirm input"
|
||||
},
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-deletion button[type=submit]"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=pending]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "completed",
|
||||
"label": "Deletion confirmed",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-delete-retry"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=completed]"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "legacy",
|
||||
"label": "Legacy ended Subject",
|
||||
"path": "/w/memory-review/memory/legacy-subject",
|
||||
"goal": "Delete a legacy ended Subject without reactivating it.",
|
||||
"dataState": "Saved retired wire state, owner.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": ".subject-delete-open"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Legacy ended Subject"
|
||||
},
|
||||
{
|
||||
"id": "confirmation",
|
||||
"label": "Delete legacy Subject",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-delete-open"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=confirming]"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "non-owner",
|
||||
"label": "Nonowner Subject view",
|
||||
"path": "/w/memory-review/memory/legacy-subject?persona=reader",
|
||||
"goal": "Read Subject without presenting unauthorized deletion.",
|
||||
"dataState": "Backend delete_subjects=false; no deletion controls.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": ".subject-overview"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Nonowner details"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"processes": [
|
||||
{
|
||||
"id": "subject-delete-fixture",
|
||||
"command": "deno",
|
||||
"cwd": "..",
|
||||
"args": [
|
||||
"run",
|
||||
"-A",
|
||||
"../../web/workspace/test/subject-delete-browser/fixture-server.ts",
|
||||
"15287",
|
||||
"../../web/workspace/build"
|
||||
],
|
||||
"readyUrl": "http://127.0.0.1:15287/health"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "subject-delete-light",
|
||||
"title": "Subject deletion confirmation and retry",
|
||||
"baseUrl": "http://127.0.0.1:15287",
|
||||
"colorScheme": "light",
|
||||
"reducedMotion": "reduce",
|
||||
"personas": [
|
||||
{
|
||||
"id": "fixture",
|
||||
"label": "Authenticated owner (synthetic HTTP)",
|
||||
"auth": {
|
||||
"kind": "anonymous"
|
||||
}
|
||||
}
|
||||
],
|
||||
"viewports": [
|
||||
{
|
||||
"label": "desktop",
|
||||
"width": 1440,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"label": "narrow",
|
||||
"width": 768,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"label": "mobile",
|
||||
"width": 390,
|
||||
"height": 844
|
||||
}
|
||||
],
|
||||
"routes": [
|
||||
{
|
||||
"id": "subject",
|
||||
"label": "Subject detail",
|
||||
"path": "/w/memory-review/memory/pending-subject",
|
||||
"goal": "Review the Subject before irreversible deletion.",
|
||||
"dataState": "Active Subject, no current Worker; owner permissions from Backend projection.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": ".subject-overview"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Subject detail"
|
||||
},
|
||||
{
|
||||
"id": "confirmation",
|
||||
"label": "Irreversible deletion scope and confirmation",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-delete-open"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=confirming]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "pending",
|
||||
"label": "Fence saved; erasure unconfirmed",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".delete-confirm input"
|
||||
},
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-deletion button[type=submit]"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=pending]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "completed",
|
||||
"label": "Deletion confirmed",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-delete-retry"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=completed]"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "legacy",
|
||||
"label": "Legacy ended Subject",
|
||||
"path": "/w/memory-review/memory/legacy-subject",
|
||||
"goal": "Delete a legacy ended Subject without reactivating it.",
|
||||
"dataState": "Saved retired wire state, owner.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": ".subject-delete-open"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Legacy ended Subject"
|
||||
},
|
||||
{
|
||||
"id": "confirmation",
|
||||
"label": "Delete legacy Subject",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": ".subject-delete-open"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[data-delete-state=confirming]"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "non-owner",
|
||||
"label": "Nonowner Subject view",
|
||||
"path": "/w/memory-review/memory/legacy-subject?persona=reader",
|
||||
"goal": "Read Subject without presenting unauthorized deletion.",
|
||||
"dataState": "Backend delete_subjects=false; no deletion controls.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": ".subject-overview"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Nonowner details"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"processes": [
|
||||
{
|
||||
"id": "subject-delete-fixture",
|
||||
"command": "deno",
|
||||
"cwd": "..",
|
||||
"args": [
|
||||
"run",
|
||||
"-A",
|
||||
"../../web/workspace/test/subject-delete-browser/fixture-server.ts",
|
||||
"15287",
|
||||
"../../web/workspace/build"
|
||||
],
|
||||
"readyUrl": "http://127.0.0.1:15287/health"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
"dev": "deno run -A npm:vite@8.3.2 dev",
|
||||
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
||||
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
|
||||
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
||||
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/subject-delete.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
||||
"test:journal:backend-contract": "cd ../.. && mkdir -p target && cargo test -p yoi-workspace-server --lib journal_browser_web_contract -- --nocapture > target/journal-browser-contract.log && cd web/workspace && deno run --allow-read=../../target/journal-browser-contract.log test/journal-backend-contract.ts ../../target/journal-browser-contract.log",
|
||||
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
|
||||
"test:browser:activation": "mkdir -p ../../target/web-ux/tmp && TMPDIR=../../target/web-ux/tmp deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/activation-browser/activation-shell.browser.test.ts",
|
||||
|
||||
Generated
+7
@@ -3,6 +3,7 @@
|
||||
"specifiers": {
|
||||
"jsr:@std/assert@*": "1.0.19",
|
||||
"jsr:@std/internal@^1.0.12": "1.0.14",
|
||||
"jsr:@std/path@1.1.4": "1.1.4",
|
||||
"npm:@codemirror/autocomplete@6.20.3": "6.20.3",
|
||||
"npm:@codemirror/commands@6.11.1": "6.11.1",
|
||||
"npm:@codemirror/language@6.12.4": "6.12.4",
|
||||
@@ -42,6 +43,12 @@
|
||||
},
|
||||
"@std/internal@1.0.14": {
|
||||
"integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7"
|
||||
},
|
||||
"@std/path@1.1.4": {
|
||||
"integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5",
|
||||
"dependencies": [
|
||||
"jsr:@std/internal"
|
||||
]
|
||||
}
|
||||
},
|
||||
"npm": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
|
||||
// Input: openapi/server-api.json
|
||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
||||
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||
// Output: web/workspace/src/lib/generated/companion-api.ts
|
||||
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
|
||||
|
||||
@@ -52,6 +52,10 @@ export type WorkspacePermissionSummary = {
|
||||
manage_secrets: boolean;
|
||||
manage_runtimes: boolean;
|
||||
delete_workspace: boolean;
|
||||
/**
|
||||
* Explicit Subject deletion authority for the current authenticated actor.
|
||||
*/
|
||||
delete_subjects: boolean;
|
||||
};
|
||||
|
||||
export type WorkspaceDeletionState =
|
||||
|
||||
@@ -118,6 +118,14 @@ export type SubjektivSubjectCreateRequest = {
|
||||
behavior_md?: string;
|
||||
};
|
||||
|
||||
export type SubjektivSubjectDeleteRequest = { operation_id: string };
|
||||
|
||||
export type SubjektivSubjectDeleteResponse = {
|
||||
subject_id: string;
|
||||
operation_id: string;
|
||||
completed: boolean;
|
||||
};
|
||||
|
||||
export type SubjektivSubjectBehaviorUpdateRequest = {
|
||||
expected_behavior_md: string;
|
||||
behavior_md: string;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
|
||||
// Input: openapi/server-api.json
|
||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
||||
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
|
||||
// Output: web/workspace/src/lib/generated/repository-api.ts
|
||||
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
|
||||
// Input: openapi/server-api.json
|
||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
||||
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||
// Output: web/workspace/src/lib/generated/runtime-api.ts
|
||||
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
|
||||
// Input: openapi/server-api.json
|
||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
||||
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||
// Output: web/workspace/src/lib/generated/ticket-api.ts
|
||||
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
|
||||
// Input: openapi/server-api.json
|
||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
||||
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
|
||||
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
|
||||
|
||||
@@ -482,6 +482,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary {
|
||||
"manage_secrets",
|
||||
"manage_runtimes",
|
||||
"delete_workspace",
|
||||
"delete_subjects",
|
||||
],
|
||||
path,
|
||||
);
|
||||
@@ -492,6 +493,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary {
|
||||
),
|
||||
manage_secrets: boolean(item.manage_secrets, `${path}.manage_secrets`),
|
||||
manage_runtimes: boolean(item.manage_runtimes, `${path}.manage_runtimes`),
|
||||
delete_subjects: boolean(item.delete_subjects, `${path}.delete_subjects`),
|
||||
delete_workspace: boolean(
|
||||
item.delete_workspace,
|
||||
`${path}.delete_workspace`,
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
<script lang="ts">
|
||||
import { untrack } from 'svelte';
|
||||
import { deleteSubjektivSubject, SubjektivSubjectDeleteError } from './api.ts';
|
||||
import { clearSubjectDeleteAttempt, readSubjectDeleteAttempt, saveSubjectDeleteAttempt, type SubjectDeleteAttempt } from './subject-delete.ts';
|
||||
import type { SubjektivSubjectResponse } from '#lib/generated/memory-api.ts';
|
||||
import { workspaceRoute } from '#lib/workspace/api/http.ts';
|
||||
|
||||
let { workspaceId, subjectId, subject, onstarted }: {
|
||||
workspaceId: string;
|
||||
subjectId: string;
|
||||
subject: SubjektivSubjectResponse | null;
|
||||
onstarted: () => void;
|
||||
} = $props();
|
||||
let attempt = $state<SubjectDeleteAttempt | null>(null);
|
||||
let phase = $state<'idle' | 'confirming' | 'running' | 'pending' | 'unknown' | 'failed' | 'completed'>('idle');
|
||||
let confirmed = $state(false);
|
||||
let error = $state<string | null>(null);
|
||||
let storageError = $state(false);
|
||||
let identity = '';
|
||||
let generation = 0;
|
||||
|
||||
$effect(() => {
|
||||
const next = JSON.stringify([workspaceId, subjectId]);
|
||||
untrack(() => {
|
||||
if (identity === next) return;
|
||||
identity = next;
|
||||
generation += 1;
|
||||
attempt = null;
|
||||
phase = 'idle';
|
||||
confirmed = false;
|
||||
error = null;
|
||||
storageError = false;
|
||||
try {
|
||||
attempt = readSubjectDeleteAttempt(sessionStorage, workspaceId, subjectId);
|
||||
if (attempt) {
|
||||
phase = 'unknown';
|
||||
onstarted();
|
||||
}
|
||||
} catch (cause) {
|
||||
storageError = true;
|
||||
error = cause instanceof Error ? cause.message : 'Cannot read the saved deletion request. Do not start a new request.';
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
async function submit(): Promise<void> {
|
||||
if (phase === 'running' || phase === 'completed' || storageError || (!attempt && (!subject || !confirmed))) return;
|
||||
const currentGeneration = generation;
|
||||
const targetWorkspace = workspaceId;
|
||||
const targetSubject = subjectId;
|
||||
const request = attempt ?? { subjectId: targetSubject, operationId: crypto.randomUUID() };
|
||||
const current = () => currentGeneration === generation && targetWorkspace === workspaceId && targetSubject === subjectId;
|
||||
// 保存に失敗したら送らない。応答喪失後に新しいkeyを作る経路を残さない。
|
||||
try { saveSubjectDeleteAttempt(sessionStorage, targetWorkspace, request); }
|
||||
catch {
|
||||
error = 'The retry identity could not be saved in this tab. No deletion request was sent. Allow session storage and try again.';
|
||||
return;
|
||||
}
|
||||
attempt = request;
|
||||
phase = 'running';
|
||||
error = null;
|
||||
onstarted();
|
||||
try {
|
||||
const result = await deleteSubjektivSubject(fetch, targetWorkspace, targetSubject, { operation_id: request.operationId });
|
||||
// 対象変更後の遅延応答は新対象を更新せず、成功した旧対象の再送入力だけ消去する。
|
||||
if (result.completed) {
|
||||
try { clearSubjectDeleteAttempt(sessionStorage, targetWorkspace, targetSubject); }
|
||||
catch { /* 消去できなくても残るのは同一keyだけ。本文や成功状態は保存しない。 */ }
|
||||
}
|
||||
if (!current()) return;
|
||||
phase = result.completed ? 'completed' : 'pending';
|
||||
} catch (cause) {
|
||||
if (!current()) return;
|
||||
phase = cause instanceof SubjektivSubjectDeleteError && cause.kind === 'rejected' ? 'failed' : 'unknown';
|
||||
error = cause instanceof Error ? cause.message : 'The deletion outcome is unknown.';
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
{#if subject || attempt || storageError}
|
||||
<section class="subject-deletion" aria-labelledby="subject-delete-heading" data-delete-state={phase}>
|
||||
<header>
|
||||
<h2 id="subject-delete-heading">Delete Subject</h2>
|
||||
{#if phase === 'idle' && !storageError}
|
||||
<button type="button" class="workspace-secondary-button subject-delete-open" onclick={() => { phase = 'confirming'; }}>Delete Subject…</button>
|
||||
{/if}
|
||||
</header>
|
||||
{#if phase === 'confirming' || attempt}
|
||||
<p class="delete-target">Target: <strong>{subject?.role ?? subjectId}</strong> · <code>{subjectId}</code></p>
|
||||
{/if}
|
||||
{#if phase === 'confirming'}
|
||||
<form onsubmit={(event) => { event.preventDefault(); void submit(); }}>
|
||||
<p><strong>This is irreversible.</strong> This Subject, its behavior, Memory, change history, staging, Surface, Journal, permissions, delegations, Writer/Session associations, and Subject Job inputs/results will be deleted. Connected and delegated Workers and Jobs must stop before completion can be confirmed.</p>
|
||||
<p>Original Worker/Session text, existing archives and Workdirs, Tickets, and other separately retained resources are <strong>not deleted</strong>. Minimal deletion retry/audit metadata is retained. Other Subjects and Workspaces are not affected.</p>
|
||||
<label class="delete-confirm"><input type="checkbox" bind:checked={confirmed} /> I have checked the target, deletion scope, retained data, and irreversible effect.</label>
|
||||
<div class="delete-actions">
|
||||
<button type="submit" class="workspace-danger-button" disabled={!confirmed}>Permanently delete Subject</button>
|
||||
<button type="button" class="workspace-secondary-button" onclick={() => { phase = 'idle'; confirmed = false; }}>Cancel</button>
|
||||
</div>
|
||||
</form>
|
||||
{:else if phase === 'running'}
|
||||
<p role="status">Deleting… Waiting for stop and erasure confirmation. Do not treat this as completed.</p>
|
||||
{:else if phase === 'pending'}
|
||||
<p role="status"><strong>Deletion started; completion unconfirmed.</strong> The deletion fence is saved and the Subject is hidden from reads/lists. External stopping or erasure is not yet confirmed. Retry the same request to continue.</p>
|
||||
{:else if phase === 'unknown'}
|
||||
<p role="status"><strong>Deletion outcome unknown.</strong> The request may already have taken effect. The saved target and operation ID are retained; retry the same request, not a new deletion.</p>
|
||||
{:else if phase === 'failed'}
|
||||
<p role="alert"><strong>Deletion request rejected.</strong> Completion is not confirmed. Resolve the reported condition, then retry the same request.</p>
|
||||
{:else if phase === 'completed'}
|
||||
<p role="status"><strong>Subject deletion completed.</strong> Backend confirmed stopping and deletion of Subject-owned data. Separately retained Worker/Session text, archives, Workdirs, and Tickets remain.</p>
|
||||
<a href={workspaceRoute(workspaceId, '/memory')}>Back to subjects →</a>
|
||||
{/if}
|
||||
{#if error}<p class="workspace-callout is-error" role="alert">{error}</p>{/if}
|
||||
{#if attempt && phase !== 'completed'}
|
||||
<details><summary>Deletion request identity</summary><code>{attempt.operationId}</code><p>This tab retains the target and same request ID across reloads, even when GET can no longer find the Subject.</p></details>
|
||||
{#if phase === 'pending' || phase === 'unknown' || phase === 'failed'}
|
||||
<button type="button" class="workspace-secondary-button subject-delete-retry" onclick={() => { void submit(); }}>Retry same deletion request</button>
|
||||
{/if}
|
||||
{/if}
|
||||
</section>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.subject-deletion { min-width: 0; }
|
||||
header { display: flex; align-items: center; justify-content: space-between; gap: .75rem; flex-wrap: wrap; }
|
||||
h2 { font-size: var(--font-size-body); margin: 0; }
|
||||
p { line-height: 1.6; }
|
||||
code, .delete-target { overflow-wrap: anywhere; }
|
||||
.delete-confirm { display: flex; align-items: flex-start; gap: .6rem; line-height: 1.5; }
|
||||
input { margin-top: .25rem; flex: 0 0 auto; }
|
||||
.delete-actions { display: flex; gap: .75rem; flex-wrap: wrap; margin-top: 1rem; }
|
||||
details { margin-block: 1rem; }
|
||||
summary { cursor: pointer; }
|
||||
</style>
|
||||
@@ -29,6 +29,8 @@ import type {
|
||||
SubjektivResidentSurfaceAvailability,
|
||||
SubjektivResidentSurfaceResponse,
|
||||
SubjektivResidentSurfaceSnapshot,
|
||||
SubjektivSubjectDeleteRequest,
|
||||
SubjektivSubjectDeleteResponse,
|
||||
SubjektivSubjectBehaviorUpdateRequest,
|
||||
SubjektivSubjectCreateRequest,
|
||||
SubjektivSubjectListResponse,
|
||||
@@ -158,6 +160,70 @@ function validateSubjectBehavior(behaviorMd: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
export class SubjektivSubjectDeleteError extends Error {
|
||||
constructor(
|
||||
readonly kind: "rejected" | "unknown_outcome",
|
||||
message: string,
|
||||
readonly status: number | null = null,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "SubjektivSubjectDeleteError";
|
||||
}
|
||||
}
|
||||
|
||||
export function parseSubjektivSubjectDeleteResponse(
|
||||
value: unknown,
|
||||
subjectId: string,
|
||||
operationId: string,
|
||||
): SubjektivSubjectDeleteResponse {
|
||||
const record = strictRecord(value, ["subject_id", "operation_id", "completed"], "Subject deletion");
|
||||
const response = {
|
||||
subject_id: requiredIdentifier(record, "subject_id"),
|
||||
operation_id: requiredIdentifier(record, "operation_id"),
|
||||
completed: requiredBoolean(record, "completed"),
|
||||
};
|
||||
if (response.subject_id !== subjectId || response.operation_id !== operationId) {
|
||||
invalid("Subject deletion response does not match the request");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
/** HTTP success can confirm only the deletion fence. Every retry retains this request. */
|
||||
export async function deleteSubjektivSubject(
|
||||
fetchFn: typeof fetch,
|
||||
workspaceId: string,
|
||||
subjectId: string,
|
||||
request: SubjektivSubjectDeleteRequest,
|
||||
): Promise<SubjektivSubjectDeleteResponse> {
|
||||
identifier(request.operation_id, "operation_id");
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetchFn(workspaceApiPath(workspaceId, `/subjektiv/subjects/${encodeURIComponent(subjectId)}`), {
|
||||
method: "DELETE",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ operation_id: request.operation_id }),
|
||||
});
|
||||
} catch {
|
||||
throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion outcome is unknown. Retry the same request.");
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new SubjektivSubjectDeleteError(
|
||||
response.status >= 500 ? "unknown_outcome" : "rejected",
|
||||
await subjektivSubjectCreateErrorMessage(response),
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
try {
|
||||
return parseSubjektivSubjectDeleteResponse(
|
||||
await readBoundedJson(response, MEMORY_API_LIMITS.maxResponseBytes),
|
||||
subjectId,
|
||||
request.operation_id,
|
||||
);
|
||||
} catch {
|
||||
throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion response could not be confirmed. Retry the same request.");
|
||||
}
|
||||
}
|
||||
|
||||
export async function createSubjektivSubject(
|
||||
fetchFn: typeof fetch,
|
||||
workspaceId: string,
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// 削除開始後はGETから対象が消えるため、同じtabでの再読込みにも依頼IDを保持する。
|
||||
// Subject本文は保存しない。これは削除結果の正本ではなく、安全な再送用の最小入力。
|
||||
export type SubjectDeleteAttempt = {
|
||||
subjectId: string;
|
||||
operationId: string;
|
||||
};
|
||||
export function subjectDeleteStorageKey(
|
||||
workspaceId: string,
|
||||
subjectId: string,
|
||||
): string {
|
||||
return `yoi.subject-delete:${JSON.stringify([workspaceId, subjectId])}`;
|
||||
}
|
||||
export function readSubjectDeleteAttempt(
|
||||
storage: Pick<Storage, "getItem">,
|
||||
workspaceId: string,
|
||||
subjectId: string,
|
||||
): SubjectDeleteAttempt | null {
|
||||
const raw = storage.getItem(subjectDeleteStorageKey(workspaceId, subjectId));
|
||||
if (raw === null) return null;
|
||||
const value = JSON.parse(raw);
|
||||
if (
|
||||
!value || value.subjectId !== subjectId ||
|
||||
typeof value.operationId !== "string" ||
|
||||
value.operationId.length === 0 ||
|
||||
Object.keys(value).some((key) =>
|
||||
key !== "subjectId" && key !== "operationId"
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
"The saved deletion request cannot be read. Do not start another deletion request.",
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
export function saveSubjectDeleteAttempt(
|
||||
storage: Pick<Storage, "setItem">,
|
||||
workspaceId: string,
|
||||
attempt: SubjectDeleteAttempt,
|
||||
): void {
|
||||
storage.setItem(
|
||||
subjectDeleteStorageKey(workspaceId, attempt.subjectId),
|
||||
JSON.stringify(attempt),
|
||||
);
|
||||
}
|
||||
export function clearSubjectDeleteAttempt(
|
||||
storage: Pick<Storage, "removeItem">,
|
||||
workspaceId: string,
|
||||
subjectId: string,
|
||||
): void {
|
||||
storage.removeItem(subjectDeleteStorageKey(workspaceId, subjectId));
|
||||
}
|
||||
@@ -38,7 +38,7 @@
|
||||
}
|
||||
|
||||
function subjectStateLabel(state: SubjektivSubjectResponse['state']): string {
|
||||
return state === 'active' ? 'Available' : 'Retired';
|
||||
return state === 'active' ? 'Available' : 'Legacy ended Subject';
|
||||
}
|
||||
|
||||
async function openCreateForm(): Promise<void> {
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
// 実行中Workerの履歴へ反映済みであることの証明には使わない。診断値は詳細へ分離する。
|
||||
import { untrack } from 'svelte';
|
||||
import DocumentMarkdown from '#lib/workspace/markdown/DocumentMarkdown.svelte';
|
||||
import SubjectDelete from '#lib/workspace/memory/SubjectDelete.svelte';
|
||||
import { formatDate, workspaceRoute } from '#lib/workspace/api/http.ts';
|
||||
import {
|
||||
SubjektivSubjectCreateError,
|
||||
@@ -26,6 +27,7 @@
|
||||
let behaviorSaving = $state(false);
|
||||
let behaviorError = $state<string | null>(null);
|
||||
let behaviorSaveConfirmed = $state(false);
|
||||
let deletionStarted = $state(false);
|
||||
|
||||
let routeGeneration = 0;
|
||||
let subjectWorkspaceId = untrack(() => data.workspaceId);
|
||||
@@ -49,6 +51,7 @@
|
||||
behaviorSaving = false;
|
||||
behaviorError = null;
|
||||
behaviorSaveConfirmed = false;
|
||||
deletionStarted = false;
|
||||
});
|
||||
});
|
||||
|
||||
@@ -149,7 +152,7 @@
|
||||
}
|
||||
|
||||
function subjectStateLabel(state: 'active' | 'retired'): string {
|
||||
return state === 'active' ? 'Available' : 'Retired';
|
||||
return state === 'active' ? 'Available' : 'Legacy ended Subject';
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -167,9 +170,16 @@
|
||||
<h1 id="subject-heading">{subject?.role ?? data.subjectId}</h1>
|
||||
<code class="subject-id" title={data.subjectId}>{data.subjectId}</code>
|
||||
</div>
|
||||
<span class="read-only-label">Behavior editable</span>
|
||||
{#if !deletionStarted}<span class="read-only-label">Behavior editable</span>{/if}
|
||||
</header>
|
||||
|
||||
{#if data.workspace.permissions.delete_subjects}
|
||||
{#key JSON.stringify([data.workspaceId, data.subjectId])}
|
||||
<SubjectDelete workspaceId={data.workspaceId} subjectId={data.subjectId} {subject} onstarted={() => { deletionStarted = true; }} />
|
||||
{/key}
|
||||
{/if}
|
||||
|
||||
{#if !deletionStarted}
|
||||
<nav aria-label="Subjectの記録"><a href={workspaceRoute(data.workspaceId, `/memory/${encodeURIComponent(data.subjectId)}/journal`)}>Journal · 最近の出来事とSessionを探す →</a></nav>
|
||||
|
||||
{#if subject}
|
||||
@@ -388,6 +398,7 @@
|
||||
<div class="memory-state" role="status"><p>Current Memory data is unavailable.</p></div>
|
||||
{/if}
|
||||
</section>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
// 本番Web buildと合成HTTP境界を接続する。実認証・削除の永続性はBackend suiteの責務。
|
||||
import { join, resolve } from "jsr:@std/path@1.1.4";
|
||||
const [portText, build, mode] = Deno.args;
|
||||
const port = Number(portText);
|
||||
const root = resolve(import.meta.dirname!, "../../../..");
|
||||
const upstreamPort = port + 1;
|
||||
const upstream = new Deno.Command(Deno.execPath(), {
|
||||
args: [
|
||||
"run",
|
||||
"-A",
|
||||
join(root, "tools/web-ux/browser-tests/memory_document_fixture_server.ts"),
|
||||
String(upstreamPort),
|
||||
resolve(build),
|
||||
"3",
|
||||
],
|
||||
stdout: "null",
|
||||
stderr: "inherit",
|
||||
}).spawn();
|
||||
const stop = () => {
|
||||
try {
|
||||
upstream.kill();
|
||||
} catch { /* already stopped */ }
|
||||
};
|
||||
Deno.addSignalListener("SIGTERM", () => {
|
||||
stop();
|
||||
Deno.exit();
|
||||
});
|
||||
Deno.addSignalListener("SIGINT", () => {
|
||||
stop();
|
||||
Deno.exit();
|
||||
});
|
||||
const attempts = new Map<string, string>();
|
||||
const hidden = new Set<string>();
|
||||
const user = {
|
||||
user_id: "fixture-user",
|
||||
account_id: "owner",
|
||||
handle: "owner",
|
||||
display_name: "Memory owner",
|
||||
};
|
||||
const apiRoot = "/api/w/memory-review";
|
||||
Deno.serve({ hostname: "127.0.0.1", port }, async (request) => {
|
||||
const url = new URL(request.url);
|
||||
if (url.pathname === "/health") {
|
||||
try {
|
||||
return await fetch(`http://127.0.0.1:${upstreamPort}/health`);
|
||||
} catch {
|
||||
return new Response(null, { status: 503 });
|
||||
}
|
||||
}
|
||||
if (url.pathname === "/api/auth/whoami") {
|
||||
return Response.json({
|
||||
authentication_reference: "subject-delete-fixture",
|
||||
actor: { ...user, auth_method: "browser_session" },
|
||||
session: { kind: "authenticated", expires_at: "2099-01-01T00:00:00Z" },
|
||||
});
|
||||
}
|
||||
const sessionId =
|
||||
request.headers.get("cookie")?.match(/delete-review-session=([^;]+)/)
|
||||
?.[1] ?? crypto.randomUUID();
|
||||
const reader =
|
||||
request.headers.get("cookie")?.includes("subject-delete-persona=reader") ??
|
||||
false;
|
||||
const match = url.pathname.match(/\/subjektiv\/subjects\/([^/]+)(?:\/|$)/);
|
||||
const id = match ? decodeURIComponent(match[1]) : null;
|
||||
const key = JSON.stringify([sessionId, id]);
|
||||
if (request.method === "DELETE" && id) {
|
||||
if (reader) {
|
||||
return Response.json({ error: "forbidden", message: "Owner required" }, {
|
||||
status: 403,
|
||||
});
|
||||
}
|
||||
const body = await request.json();
|
||||
const prior = attempts.get(key);
|
||||
if (prior && prior !== body.operation_id) {
|
||||
return Response.json(
|
||||
{ error: "conflict", message: "Changed operation" },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
attempts.set(key, body.operation_id);
|
||||
hidden.add(key);
|
||||
if (!prior && id === "unknown-subject") {
|
||||
return Response.json({
|
||||
error: "internal",
|
||||
message: "Response lost after fence",
|
||||
}, { status: 500 });
|
||||
}
|
||||
return Response.json({
|
||||
subject_id: id,
|
||||
operation_id: body.operation_id,
|
||||
completed: !!prior,
|
||||
});
|
||||
}
|
||||
if (id && request.method === "GET" && hidden.has(key)) {
|
||||
return Response.json(
|
||||
{ error: "not_found", message: "Subject unavailable" },
|
||||
{ status: 404 },
|
||||
);
|
||||
}
|
||||
const mapped = id &&
|
||||
["pending-subject", "unknown-subject", "legacy-subject"].includes(id);
|
||||
const target = new URL(request.url);
|
||||
target.port = String(upstreamPort);
|
||||
if (mapped) {
|
||||
target.pathname = target.pathname.replace(
|
||||
encodeURIComponent(id!),
|
||||
"empty-subject",
|
||||
);
|
||||
}
|
||||
let response = await fetch(new Request(target, request));
|
||||
if (url.pathname === `${apiRoot}/workspace` || mapped) {
|
||||
const data = await response.json();
|
||||
if (url.pathname === `${apiRoot}/workspace` && mode !== "before") {
|
||||
data.permissions.delete_subjects = !reader;
|
||||
}
|
||||
if (mapped) {
|
||||
if ("id" in data) {
|
||||
data.id = id;
|
||||
data.role = `Deletion review · ${id}`;
|
||||
if (id === "legacy-subject") data.state = "retired";
|
||||
}
|
||||
if ("subject_id" in data) data.subject_id = id;
|
||||
}
|
||||
response = Response.json(data, { status: response.status });
|
||||
}
|
||||
response = new Response(response.body, {
|
||||
status: response.status,
|
||||
headers: response.headers,
|
||||
});
|
||||
const persona = url.searchParams.get("persona");
|
||||
if (persona) {
|
||||
response.headers.set(
|
||||
"set-cookie",
|
||||
`subject-delete-persona=${persona}; Path=/; HttpOnly; SameSite=Lax`,
|
||||
);
|
||||
}
|
||||
response.headers.append(
|
||||
"set-cookie",
|
||||
`delete-review-session=${sessionId}; Path=/; HttpOnly; SameSite=Lax`,
|
||||
);
|
||||
return response;
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
import { assert, assertEquals } from "@std/assert";
|
||||
import { chromium, type Page } from "playwright";
|
||||
import { join, resolve } from "@std/path";
|
||||
import {
|
||||
assertNoPageWideOverflow,
|
||||
freePort,
|
||||
waitForServer,
|
||||
} from "../../../../tools/web-ux/browser-tests/memory_document_harness.ts";
|
||||
|
||||
const root = resolve(import.meta.dirname!, "../../../..");
|
||||
const artifact = join(root, "target/web-ux/t786-browser");
|
||||
async function confirm(page: Page) {
|
||||
await page.locator(".subject-delete-open").click();
|
||||
const button = page.getByRole("button", {
|
||||
name: "Permanently delete Subject",
|
||||
exact: true,
|
||||
});
|
||||
assert(await button.isDisabled());
|
||||
await page.getByRole("checkbox").check();
|
||||
await button.click();
|
||||
}
|
||||
Deno.test({
|
||||
name:
|
||||
"Subject deletion shell: owner confirmation, pending/reload/retry, unknown/rejected and route fencing",
|
||||
sanitizeOps: false,
|
||||
sanitizeResources: false,
|
||||
fn: async () => {
|
||||
const build = await new Deno.Command(Deno.execPath(), {
|
||||
args: ["task", "build"],
|
||||
cwd: join(root, "web/workspace"),
|
||||
stdout: "null",
|
||||
stderr: "piped",
|
||||
}).output();
|
||||
assert(build.success, new TextDecoder().decode(build.stderr));
|
||||
const port = await freePort();
|
||||
const server = new Deno.Command(Deno.execPath(), {
|
||||
args: [
|
||||
"run",
|
||||
"-A",
|
||||
join(import.meta.dirname!, "fixture-server.ts"),
|
||||
String(port),
|
||||
join(root, "web/workspace/build"),
|
||||
],
|
||||
stdout: "null",
|
||||
stderr: "inherit",
|
||||
}).spawn();
|
||||
const origin = `http://127.0.0.1:${port}`;
|
||||
await Deno.mkdir(artifact, { recursive: true });
|
||||
const browser = await chromium.launch();
|
||||
try {
|
||||
await waitForServer(`${origin}/health`);
|
||||
for (const theme of ["light", "dark"] as const) {
|
||||
for (const width of [1440, 768, 390]) {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width, height: 900 },
|
||||
colorScheme: theme,
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const pageErrors: string[] = [];
|
||||
page.on("pageerror", (error) => pageErrors.push(error.message));
|
||||
const requests: unknown[] = [];
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
await page.route(
|
||||
"**/subjektiv/subjects/pending-subject",
|
||||
async (route) => {
|
||||
if (route.request().method() !== "DELETE") {
|
||||
return route.continue();
|
||||
}
|
||||
requests.push(route.request().postDataJSON());
|
||||
if (requests.length === 1) await held;
|
||||
await route.continue();
|
||||
},
|
||||
);
|
||||
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
|
||||
await page.locator(".subject-delete-open").waitFor();
|
||||
await page.screenshot({
|
||||
path: join(artifact, `${theme}-${width}-initial.png`),
|
||||
});
|
||||
await page.locator(".subject-delete-open").click();
|
||||
assert(
|
||||
await page.getByText("This is irreversible.", { exact: true })
|
||||
.isVisible(),
|
||||
);
|
||||
await assertNoPageWideOverflow(page);
|
||||
await page.screenshot({
|
||||
path: join(artifact, `${theme}-${width}-confirmation.png`),
|
||||
});
|
||||
await page.getByRole("button", { name: "Cancel", exact: true })
|
||||
.click();
|
||||
assertEquals(requests.length, 0);
|
||||
await confirm(page);
|
||||
await page.locator('[data-delete-state="running"]').waitFor();
|
||||
assertEquals(await page.locator(".subject-overview").count(), 0);
|
||||
await page.screenshot({
|
||||
path: join(artifact, `${theme}-${width}-running.png`),
|
||||
});
|
||||
release();
|
||||
await page.locator('[data-delete-state="pending"]').waitFor();
|
||||
assertEquals(requests.length, 1);
|
||||
const saved = await page.evaluate(() =>
|
||||
Object.keys(sessionStorage).filter((key) =>
|
||||
key.startsWith("yoi.subject-delete:")
|
||||
).map((key) => JSON.parse(sessionStorage.getItem(key)!))
|
||||
);
|
||||
assertEquals(saved, [{
|
||||
subjectId: "pending-subject",
|
||||
operationId: (requests[0] as { operation_id: string }).operation_id,
|
||||
}]);
|
||||
await page.screenshot({
|
||||
path: join(artifact, `${theme}-${width}-pending.png`),
|
||||
});
|
||||
await page.reload();
|
||||
await page.locator('[data-delete-state="unknown"]').waitFor();
|
||||
assertEquals(await page.locator(".subject-overview").count(), 0);
|
||||
await page.locator(".subject-delete-retry").click();
|
||||
await page.locator('[data-delete-state="completed"]').waitFor();
|
||||
assertEquals(requests.length, 2);
|
||||
assertEquals(requests[0], requests[1]);
|
||||
assertEquals(
|
||||
await page.evaluate(() =>
|
||||
Object.keys(sessionStorage).filter((key) =>
|
||||
key.startsWith("yoi.subject-delete:")
|
||||
)
|
||||
),
|
||||
[],
|
||||
);
|
||||
await page.screenshot({
|
||||
path: join(artifact, `${theme}-${width}-completed.png`),
|
||||
});
|
||||
await assertNoPageWideOverflow(page);
|
||||
assertEquals(pageErrors, []);
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
for (const subject of ["unknown-subject", "legacy-subject"]) {
|
||||
const page = await browser.newPage();
|
||||
const requests: unknown[] = [];
|
||||
page.on("request", (request) => {
|
||||
if (request.method() === "DELETE") {
|
||||
requests.push(request.postDataJSON());
|
||||
}
|
||||
});
|
||||
await page.goto(`${origin}/w/memory-review/memory/${subject}`);
|
||||
await page.locator(".subject-delete-open").waitFor();
|
||||
if (subject === "legacy-subject") {
|
||||
assert(
|
||||
await page.getByText("Legacy ended Subject", { exact: true })
|
||||
.isVisible(),
|
||||
);
|
||||
}
|
||||
await confirm(page);
|
||||
await page.locator(
|
||||
`[data-delete-state="${
|
||||
subject === "unknown-subject" ? "unknown" : "pending"
|
||||
}"]`,
|
||||
).waitFor();
|
||||
await page.locator(".subject-delete-retry").click();
|
||||
await page.locator('[data-delete-state="completed"]').waitFor();
|
||||
assertEquals(requests.length, 2);
|
||||
assertEquals(requests[0], requests[1]);
|
||||
await page.close();
|
||||
}
|
||||
// 4xxでもkeyを捨てない。権限の変化を新しい依頼へ置換しない。
|
||||
{
|
||||
const page = await browser.newPage();
|
||||
const requests: unknown[] = [];
|
||||
await page.route(
|
||||
"**/subjektiv/subjects/pending-subject",
|
||||
async (route) => {
|
||||
if (route.request().method() !== "DELETE") return route.continue();
|
||||
requests.push(route.request().postDataJSON());
|
||||
if (requests.length === 1) {
|
||||
await route.fulfill({
|
||||
status: 403,
|
||||
json: { error: "forbidden", message: "Permission changed" },
|
||||
});
|
||||
} else {await route.fulfill({
|
||||
json: {
|
||||
subject_id: "pending-subject",
|
||||
operation_id: (requests[0] as any).operation_id,
|
||||
completed: true,
|
||||
},
|
||||
});}
|
||||
},
|
||||
);
|
||||
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
|
||||
await confirm(page);
|
||||
await page.locator('[data-delete-state="failed"]').waitFor();
|
||||
await page.screenshot({ path: join(artifact, "rejected.png") });
|
||||
await page.locator(".subject-delete-retry").click();
|
||||
await page.locator('[data-delete-state="completed"]').waitFor();
|
||||
assertEquals(requests[0], requests[1]);
|
||||
await page.close();
|
||||
}
|
||||
// 同一SPA内で他対象へ移動してから旧応答が届いても、新対象を消さない。
|
||||
{
|
||||
const page = await browser.newPage();
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
await page.route(
|
||||
"**/subjektiv/subjects/pending-subject",
|
||||
async (route) => {
|
||||
if (route.request().method() !== "DELETE") return route.continue();
|
||||
const body = route.request().postDataJSON();
|
||||
await held;
|
||||
await route.fulfill({
|
||||
json: {
|
||||
subject_id: "pending-subject",
|
||||
operation_id: body.operation_id,
|
||||
completed: true,
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
|
||||
await confirm(page);
|
||||
await page.locator('[data-delete-state="running"]').waitFor();
|
||||
await page.getByRole("link", { name: "← All subjects", exact: true })
|
||||
.click();
|
||||
await page.locator(".subject-list a").first().click();
|
||||
await page.locator(".subject-overview").waitFor();
|
||||
const target = page.url();
|
||||
const response = page.waitForResponse((r) =>
|
||||
r.request().method() === "DELETE"
|
||||
);
|
||||
release();
|
||||
await response;
|
||||
assertEquals(page.url(), target);
|
||||
assertEquals(
|
||||
await page.locator('[data-delete-state="completed"]').count(),
|
||||
0,
|
||||
);
|
||||
assertEquals(await page.locator(".subject-overview").count(), 1);
|
||||
await page.close();
|
||||
}
|
||||
{
|
||||
const page = await browser.newPage();
|
||||
await page.goto(
|
||||
`${origin}/w/memory-review/memory/legacy-subject?persona=reader`,
|
||||
);
|
||||
await page.locator(".subject-overview").waitFor();
|
||||
assertEquals(await page.locator(".subject-deletion").count(), 0);
|
||||
await page.screenshot({ path: join(artifact, "non-owner.png") });
|
||||
await page.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
server.kill("SIGTERM");
|
||||
await server.status;
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,152 @@
|
||||
import {
|
||||
deleteSubjektivSubject,
|
||||
parseSubjektivSubjectDeleteResponse,
|
||||
SubjektivSubjectDeleteError,
|
||||
} from "../src/lib/workspace/memory/api.ts";
|
||||
import {
|
||||
clearSubjectDeleteAttempt,
|
||||
readSubjectDeleteAttempt,
|
||||
saveSubjectDeleteAttempt,
|
||||
subjectDeleteStorageKey,
|
||||
} from "../src/lib/workspace/memory/subject-delete.ts";
|
||||
|
||||
declare const Deno: {
|
||||
test(name: string, fn: () => void | Promise<void>): void;
|
||||
};
|
||||
|
||||
function equal(actual: unknown, expected: unknown) {
|
||||
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
||||
throw new Error(
|
||||
`Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
Deno.test("Subject deletion uses DELETE with the same request after fence-only success", async () => {
|
||||
const calls: unknown[] = [];
|
||||
const fetcher = ((input, init) => {
|
||||
calls.push([input, init]);
|
||||
return Promise.resolve(
|
||||
Response.json({
|
||||
subject_id: "s/1",
|
||||
operation_id: "same-operation",
|
||||
completed: calls.length > 1,
|
||||
}),
|
||||
);
|
||||
}) as typeof fetch;
|
||||
for (const completed of [false, true]) {
|
||||
equal(
|
||||
await deleteSubjektivSubject(fetcher, "w/1", "s/1", {
|
||||
operation_id: "same-operation",
|
||||
}),
|
||||
{ subject_id: "s/1", operation_id: "same-operation", completed },
|
||||
);
|
||||
}
|
||||
equal(calls[0], calls[1]);
|
||||
equal(calls[0], ["/api/w/w%2F1/subjektiv/subjects/s%2F1", {
|
||||
method: "DELETE",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: '{"operation_id":"same-operation"}',
|
||||
}]);
|
||||
});
|
||||
Deno.test("Subject deletion never confirms a malformed or wrong-target response", () => {
|
||||
for (
|
||||
const payload of [
|
||||
{ subject_id: "other", operation_id: "op", completed: true },
|
||||
{ subject_id: "s", operation_id: "other", completed: true },
|
||||
{ subject_id: "s", operation_id: "op", completed: "true" },
|
||||
{ subject_id: "s", operation_id: "op" },
|
||||
{ subject_id: "s", operation_id: "op", completed: true, extra: 1 },
|
||||
]
|
||||
) {
|
||||
try {
|
||||
parseSubjektivSubjectDeleteResponse(payload, "s", "op");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
throw new Error("Accepted an unconfirmed response");
|
||||
}
|
||||
});
|
||||
Deno.test("Subject deletion classifies rejection separately from unknown transport/server/response outcomes", async () => {
|
||||
for (
|
||||
const item of [
|
||||
{
|
||||
fetcher: () => Promise.reject(new Error("lost")),
|
||||
kind: "unknown_outcome",
|
||||
},
|
||||
{
|
||||
fetcher: () =>
|
||||
Promise.resolve(
|
||||
Response.json({ error: "forbidden" }, { status: 403 }),
|
||||
),
|
||||
kind: "rejected",
|
||||
},
|
||||
{
|
||||
fetcher: () =>
|
||||
Promise.resolve(
|
||||
Response.json({ error: "conflict" }, { status: 409 }),
|
||||
),
|
||||
kind: "rejected",
|
||||
},
|
||||
{
|
||||
fetcher: () =>
|
||||
Promise.resolve(
|
||||
Response.json({ error: "uncertain" }, { status: 500 }),
|
||||
),
|
||||
kind: "unknown_outcome",
|
||||
},
|
||||
{
|
||||
fetcher: () =>
|
||||
Promise.resolve(
|
||||
Response.json({
|
||||
subject_id: "wrong",
|
||||
operation_id: "op",
|
||||
completed: true,
|
||||
}),
|
||||
),
|
||||
kind: "unknown_outcome",
|
||||
},
|
||||
]
|
||||
) {
|
||||
try {
|
||||
await deleteSubjektivSubject(item.fetcher as typeof fetch, "w", "s", {
|
||||
operation_id: "op",
|
||||
});
|
||||
} catch (error) {
|
||||
if (!(error instanceof SubjektivSubjectDeleteError)) throw error;
|
||||
equal(error.kind, item.kind);
|
||||
continue;
|
||||
}
|
||||
throw new Error("Unexpected success");
|
||||
}
|
||||
});
|
||||
Deno.test("Retry input survives reload independently of Subject GET and is isolated by Workspace/Subject", () => {
|
||||
const values = new Map<string, string>();
|
||||
const storage = {
|
||||
getItem: (key: string) => values.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => {
|
||||
values.set(key, value);
|
||||
},
|
||||
removeItem: (key: string) => {
|
||||
values.delete(key);
|
||||
},
|
||||
};
|
||||
const attempt = {
|
||||
subjectId: "s",
|
||||
operationId: "same-op",
|
||||
};
|
||||
saveSubjectDeleteAttempt(storage, "w", attempt);
|
||||
equal(readSubjectDeleteAttempt(storage, "w", "s"), attempt);
|
||||
equal(readSubjectDeleteAttempt(storage, "other", "s"), null);
|
||||
equal(readSubjectDeleteAttempt(storage, "w", "other"), null);
|
||||
saveSubjectDeleteAttempt(storage, "other", attempt);
|
||||
clearSubjectDeleteAttempt(storage, "w", "s");
|
||||
equal(readSubjectDeleteAttempt(storage, "w", "s"), null);
|
||||
equal(readSubjectDeleteAttempt(storage, "other", "s"), attempt);
|
||||
values.set(subjectDeleteStorageKey("w", "s"), '{"subjectId":"other"}');
|
||||
try {
|
||||
readSubjectDeleteAttempt(storage, "w", "s");
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
throw new Error("Corrupt retry input allowed new deletion");
|
||||
});
|
||||
Reference in New Issue
Block a user