feat: replace Subject retirement with durable explicit deletion
This commit is contained in:
@@ -1193,8 +1193,40 @@ pub struct SubjectCreateRecoveryResponse {
|
|||||||
pub operation_id: String,
|
pub operation_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Host requests this only after durably fencing the Subject in Backend authority.
|
||||||
|
/// Permanently exclude the Worker from creation and restoration, and confirm actual
|
||||||
|
/// execution shutdown. This does not delete Worker or Session records.
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SubjectDeleteWorkerRequest {
|
||||||
|
pub workspace_id: String,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub create_fingerprint: String,
|
||||||
|
pub operation_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Durable admission exclusion plus confirmed execution shutdown/absence. Errors,
|
||||||
|
/// timeouts and lost responses are unknown outcomes; retry the same full request.
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SubjectDeleteWorkerResponse {
|
||||||
|
pub workspace_id: String,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub create_fingerprint: String,
|
||||||
|
pub operation_id: String,
|
||||||
|
/// The canonical live Worker/Session/diagnostic aggregate is physically absent
|
||||||
|
/// under the permanent creation fence. False is not a deletion receipt.
|
||||||
|
pub originals_absent: bool,
|
||||||
|
}
|
||||||
|
|
||||||
#[api(reqwest, axum)]
|
#[api(reqwest, axum)]
|
||||||
pub trait RuntimeApi {
|
pub trait RuntimeApi {
|
||||||
|
#[post("/v1/subject-delete-worker", status = 200, error_status = 400)]
|
||||||
|
async fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
#[body] request: SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<SubjectDeleteWorkerResponse, RuntimeApiError>;
|
||||||
|
|
||||||
#[post("/v1/subject-create-recovery", status = 200, error_status = 400)]
|
#[post("/v1/subject-create-recovery", status = 200, error_status = 400)]
|
||||||
async fn recover_subject_create(
|
async fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
@@ -1755,6 +1787,19 @@ mod tests {
|
|||||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
|
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
|
||||||
Err(test_error(501))
|
Err(test_error(501))
|
||||||
}
|
}
|
||||||
|
async fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<SubjectDeleteWorkerResponse, RuntimeApiError> {
|
||||||
|
Ok(SubjectDeleteWorkerResponse {
|
||||||
|
originals_absent: true,
|
||||||
|
workspace_id: request.workspace_id,
|
||||||
|
worker_id: request.worker_id,
|
||||||
|
create_fingerprint: request.create_fingerprint,
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn recover_subject_create(
|
async fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
request: SubjectCreateRecoveryRequest,
|
request: SubjectCreateRecoveryRequest,
|
||||||
@@ -1934,6 +1979,19 @@ mod tests {
|
|||||||
assert_eq!(response.create_fingerprint, recovery.create_fingerprint);
|
assert_eq!(response.create_fingerprint, recovery.create_fingerprint);
|
||||||
assert_eq!(response.operation_id, recovery.operation_id);
|
assert_eq!(response.operation_id, recovery.operation_id);
|
||||||
|
|
||||||
|
let deletion = SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: "workspace-a".into(),
|
||||||
|
worker_id: WorkerId::now_v7(),
|
||||||
|
create_fingerprint: "deleted-create".into(),
|
||||||
|
operation_id: "delete-1".into(),
|
||||||
|
};
|
||||||
|
let receipt = client
|
||||||
|
.subject_delete_worker(deletion.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut expected = serde_json::to_value(deletion).unwrap();
|
||||||
|
expected["originals_absent"] = true.into();
|
||||||
|
assert_eq!(serde_json::to_value(receipt).unwrap(), expected);
|
||||||
let summary = client.runtime_summary().await.unwrap();
|
let summary = client.runtime_summary().await.unwrap();
|
||||||
assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip"));
|
assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip"));
|
||||||
let workers = client
|
let workers = client
|
||||||
|
|||||||
@@ -338,6 +338,8 @@ impl_openapi_schema!(
|
|||||||
MemoryConsolidationResponse,
|
MemoryConsolidationResponse,
|
||||||
SubjektivSubjectCreateRequest,
|
SubjektivSubjectCreateRequest,
|
||||||
SubjektivSubjectBehaviorUpdateRequest,
|
SubjektivSubjectBehaviorUpdateRequest,
|
||||||
|
SubjektivSubjectDeleteRequest,
|
||||||
|
SubjektivSubjectDeleteResponse,
|
||||||
SubjektivSubjectListQuery,
|
SubjektivSubjectListQuery,
|
||||||
SubjektivSubjectListResponse,
|
SubjektivSubjectListResponse,
|
||||||
SubjektivSubjectResponse,
|
SubjektivSubjectResponse,
|
||||||
@@ -1527,19 +1529,21 @@ pub trait ServerApi {
|
|||||||
#[path] memory_id: String,
|
#[path] memory_id: String,
|
||||||
#[query] query: SubjektivMemoryChangesQuery,
|
#[query] query: SubjektivMemoryChangesQuery,
|
||||||
) -> Result<SubjektivMemoryListChangesResponse, RepositoryApiError>;
|
) -> Result<SubjektivMemoryListChangesResponse, RepositoryApiError>;
|
||||||
#[post(
|
#[delete(
|
||||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire",
|
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}",
|
||||||
status = 200,
|
status = 200,
|
||||||
error_status = 400,
|
error_status = 400,
|
||||||
additional_error_statuses = [401, 403, 404, 409, 500],
|
additional_error_statuses = [401, 403, 404, 409, 500],
|
||||||
bearer_auth = true,
|
bearer_auth = true,
|
||||||
browser_auth = true,
|
browser_auth = true,
|
||||||
)]
|
)]
|
||||||
async fn subjektiv_subject_retire(
|
async fn subjektiv_subject_delete(
|
||||||
&self,
|
&self,
|
||||||
|
#[extension] context: ServerRequestContext,
|
||||||
#[path] workspace_id: String,
|
#[path] workspace_id: String,
|
||||||
#[path] subject_id: String,
|
#[path] subject_id: String,
|
||||||
) -> Result<SubjektivSubjectResponse, RepositoryApiError>;
|
#[body] request: SubjektivSubjectDeleteRequest,
|
||||||
|
) -> Result<SubjektivSubjectDeleteResponse, RepositoryApiError>;
|
||||||
#[post(
|
#[post(
|
||||||
"/api/w/{workspace_id}/subjektiv/staging",
|
"/api/w/{workspace_id}/subjektiv/staging",
|
||||||
status = 200,
|
status = 200,
|
||||||
@@ -4955,6 +4959,8 @@ pub struct WorkspacePermissionSummary {
|
|||||||
pub manage_secrets: bool,
|
pub manage_secrets: bool,
|
||||||
pub manage_runtimes: bool,
|
pub manage_runtimes: bool,
|
||||||
pub delete_workspace: bool,
|
pub delete_workspace: bool,
|
||||||
|
/// Explicit Subject deletion authority for the current authenticated actor.
|
||||||
|
pub delete_subjects: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
@@ -5350,7 +5356,25 @@ pub struct MemoryConsolidationResponse {
|
|||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum SubjektivSubjectState {
|
pub enum SubjektivSubjectState {
|
||||||
Active,
|
Active,
|
||||||
Retired,
|
/// Saved compatibility only; not a current lifecycle operation.
|
||||||
|
#[serde(rename = "retired")]
|
||||||
|
LegacyRetired,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SubjektivSubjectDeleteRequest {
|
||||||
|
/// Stable identity retained for every retry, including unknown outcomes.
|
||||||
|
pub operation_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SubjektivSubjectDeleteResponse {
|
||||||
|
pub subject_id: String,
|
||||||
|
pub operation_id: String,
|
||||||
|
/// False confirms the deletion fence, not external stop or erasure completion.
|
||||||
|
pub completed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
@@ -10950,6 +10974,10 @@ pub fn memory_api_typescript() -> String {
|
|||||||
|
|
||||||
export type SubjektivSubjectCreateRequest = { role: string, behavior_md?: string, };
|
export type SubjektivSubjectCreateRequest = { role: string, behavior_md?: string, };
|
||||||
|
|
||||||
|
export type SubjektivSubjectDeleteRequest = { operation_id: string, };
|
||||||
|
|
||||||
|
export type SubjektivSubjectDeleteResponse = { subject_id: string, operation_id: string, completed: boolean, };
|
||||||
|
|
||||||
export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_md: string, behavior_md: string, };
|
export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_md: string, behavior_md: string, };
|
||||||
|
|
||||||
export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, state: SubjektivSubjectState, memory_fingerprint: string, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, };
|
export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, state: SubjektivSubjectState, memory_fingerprint: string, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, };
|
||||||
@@ -13483,7 +13511,8 @@ mod tests {
|
|||||||
"manage_repositories": true,
|
"manage_repositories": true,
|
||||||
"manage_secrets": true,
|
"manage_secrets": true,
|
||||||
"manage_runtimes": true,
|
"manage_runtimes": true,
|
||||||
"delete_workspace": true
|
"delete_workspace": true,
|
||||||
|
"delete_subjects": true
|
||||||
},
|
},
|
||||||
"extension_points": {
|
"extension_points": {
|
||||||
"store": "sqlite",
|
"store": "sqlite",
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
use serde_json::json;
|
||||||
|
use server_api::{
|
||||||
|
SubjektivSubjectDeleteRequest, SubjektivSubjectDeleteResponse, SubjektivSubjectState,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_delete_api_replaces_retire_and_requires_operation_identity() {
|
||||||
|
let document: serde_json::Value = serde_json::from_str(
|
||||||
|
&server_api::canonical_openapi_document()
|
||||||
|
.unwrap()
|
||||||
|
.to_json()
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let paths = &document["paths"];
|
||||||
|
assert!(
|
||||||
|
paths
|
||||||
|
.get("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let operation = &paths["/api/w/{workspace_id}/subjektiv/subjects/{subject_id}"]["delete"];
|
||||||
|
assert_eq!(operation["operationId"], "subjektiv_subject_delete");
|
||||||
|
assert_eq!(operation["requestBody"]["required"], true);
|
||||||
|
assert!(serde_json::from_value::<SubjektivSubjectDeleteRequest>(json!({})).is_err());
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<SubjektivSubjectDeleteRequest>(
|
||||||
|
json!({"operation_id":"op", "force":true})
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
for completed in [false, true] {
|
||||||
|
let response = SubjektivSubjectDeleteResponse {
|
||||||
|
subject_id: "subject".into(),
|
||||||
|
operation_id: "op".into(),
|
||||||
|
completed,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(response).unwrap(),
|
||||||
|
json!({"subject_id":"subject", "operation_id":"op", "completed":completed})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn old_subject_state_is_explicit_saved_compatibility_not_a_new_operation() {
|
||||||
|
let state: SubjektivSubjectState = serde_json::from_value(json!("retired")).unwrap();
|
||||||
|
assert_eq!(state, SubjektivSubjectState::LegacyRetired);
|
||||||
|
assert_eq!(serde_json::to_value(state).unwrap(), json!("retired"));
|
||||||
|
}
|
||||||
@@ -12,3 +12,49 @@ Hosts retain body ownership across inactivity, reserve actual execution quiescen
|
|||||||
- [`api`]: shared Host operations and bounded responses
|
- [`api`]: shared Host operations and bounded responses
|
||||||
- [`job`]: consolidation selection rules and result validation
|
- [`job`]: consolidation selection rules and result validation
|
||||||
- [`journal`]: individually committed activity operations, source coverage, retry receipts, and bounded reading
|
- [`journal`]: individually committed activity operations, source coverage, retry receipts, and bounded reading
|
||||||
|
|
||||||
|
## Explicit Subject deletion
|
||||||
|
|
||||||
|
`begin_subject_deletion(subject_id, operation_id)` commits a permanent content-free
|
||||||
|
fence in this store **before** the Host stops execution or erases external content.
|
||||||
|
The operation ID is unique within this storage scope. Exact retries return current
|
||||||
|
saved state; a different key for that Subject or reuse for another Subject conflicts.
|
||||||
|
Pending fences survive restart and a failed purge. Normal reads, writes, saved-receipt
|
||||||
|
replay, cached Host/Journal contexts and authority lookup cannot use a fenced Subject.
|
||||||
|
Subject discovery hides fenced records before pagination. SQL INSERT/UPDATE guards
|
||||||
|
cover direct and indirect owners, checking both OLD and NEW ownership on UPDATE.
|
||||||
|
Deleting a grant never restores non-Subject fallback: content-free Worker identity
|
||||||
|
fences remain, and the Host must consult them before fallback authorization.
|
||||||
|
|
||||||
|
The Host uses `subject_deletion` / `pending_subject_deletions` for recovery and
|
||||||
|
`deletion_subject_workers` / `deletion_subject_sessions` / `deletion_subject_jobs`
|
||||||
|
only as cleanup inventories, not authority or model-visible content. It must fence
|
||||||
|
creation reservations, quiesce Workers and in-flight delivery, purge Job inputs/results
|
||||||
|
and live Job execution copies, and clear resident/derived caches. Source Worker/Session
|
||||||
|
originals and pre-existing archives have independent retention lifecycles: the Workspace
|
||||||
|
Host archives source originals rather than erasing them. Those archives may contain
|
||||||
|
previously injected Memory; Workdirs, Tickets, exports and external provider copies are
|
||||||
|
also outside Subject content erasure. These boundaries must be explained to the user.
|
||||||
|
The domain rechecks after out-of-DB capture callbacks and before returning Job read
|
||||||
|
responses; already returned DTOs cannot be revoked by SQLite. Host lifecycle locking
|
||||||
|
must cover delivery/reinjection of previously fetched content as well as finish.
|
||||||
|
|
||||||
|
Only after external cleanup succeeds may the Host call
|
||||||
|
`finish_subject_deletion(subject_id, operation_id)`. One transaction deletes Subject,
|
||||||
|
Memory/history, staging/evidence, Surface snapshots/runs, permission sources/receipts,
|
||||||
|
child grants, writer/session bindings, Journal content/generations/deliveries/receipts,
|
||||||
|
links, and the Subject's `legacy_record_archive` rows. A purge error rolls back all
|
||||||
|
those deletions without undoing the separately committed fence. Ordinary history
|
||||||
|
retention and explicit Journal Session-erasure rules are unchanged for other Subjects.
|
||||||
|
Retained records contain only deletion operation/time/completion and execution/session
|
||||||
|
identities needed to prevent reuse; existing content-free Session-erasure markers and
|
||||||
|
workspace-only migration metadata remain. Other Subjects and storage scopes are untouched.
|
||||||
|
|
||||||
|
The deletion migration adds fences without erasing or reactivating existing data. Old `retired`
|
||||||
|
JSON/SQL records deserialize as `SubjectState::LegacyRetired` and remain readable
|
||||||
|
until explicitly deleted. There is no current retirement API. Normal Worker deletion
|
||||||
|
still does not implicitly delete Subject memory.
|
||||||
|
|
||||||
|
Completion means logical erasure from domain and Host-managed storage, **not** secure
|
||||||
|
physical overwriting: SQLite free pages, WAL files, filesystem snapshots, backups and
|
||||||
|
external copies can retain bytes under their own retention/erasure policies.
|
||||||
|
|||||||
@@ -116,6 +116,7 @@ impl HostOperationContext {
|
|||||||
"Host operation context belongs to another storage scope".into(),
|
"Host operation context belongs to another storage scope".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
Self::require_subject(store, &self.subject_id)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -452,3 +452,45 @@ fn query_content_condition_ignores_behavior_and_detects_aba_memory_writes() {
|
|||||||
Err(OperationError::Conflict(_))
|
Err(OperationError::Conflict(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_deletion_revalidates_cached_body_empty_batch_and_receipt_contexts() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, store) = open(temp.path(), "scope");
|
||||||
|
let subject = store
|
||||||
|
.create_subject(SubjectRole::new("companion").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let body = HostOperationContext::validated_body(&store, &subject.id, None).unwrap();
|
||||||
|
let empty =
|
||||||
|
HostOperationContext::validated_consolidation(&store, &subject.id, Some(vec![]), None)
|
||||||
|
.unwrap();
|
||||||
|
store.begin_subject_deletion(&subject.id, "delete").unwrap();
|
||||||
|
for context in [&body, &empty] {
|
||||||
|
for operation in [
|
||||||
|
query(None),
|
||||||
|
Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None }),
|
||||||
|
Op::ReceiptStatus(server_api::SubjektivMemoryReceiptStatusRequest {
|
||||||
|
receipt_id: "receipt".into(),
|
||||||
|
}),
|
||||||
|
Op::ReadCandidate(server_api::SubjektivMemoryCandidateReadRequest {
|
||||||
|
candidate_id: "candidate".into(),
|
||||||
|
}),
|
||||||
|
] {
|
||||||
|
assert!(matches!(
|
||||||
|
execute(&store, context, operation),
|
||||||
|
Err(OperationError::SubjectNotFound(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
store
|
||||||
|
.finish_subject_deletion(&subject.id, "delete")
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
execute(
|
||||||
|
&store,
|
||||||
|
&empty,
|
||||||
|
Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None })
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,419 @@
|
|||||||
|
//! Explicit Subject erasure is a two-phase Host operation. Beginning commits a permanent,
|
||||||
|
//! content-free fence before the Host stops/removes Workers and purges Jobs and execution copies.
|
||||||
|
//! Source Worker/Session originals remain under their separate archive lifecycle.
|
||||||
|
//! Failure leaves that fence and the cleanup inventory intact; the Host retries
|
||||||
|
//! the same operation ID. Finishing asserts that external cleanup succeeded and atomically erases
|
||||||
|
//! this store's content. This store cannot verify external Runtime/Job/Session cleanup itself.
|
||||||
|
//!
|
||||||
|
//! Tombstones contain no behavior, permission vectors, Memory, Journal or Session text. Worker
|
||||||
|
//! identities remain permanently fenced so removal of a grant cannot become non-Subject fallback
|
||||||
|
//! authority. Existing Journal Session erasure markers are retained to reject delayed admission;
|
||||||
|
//! they are content-free and are not a second copy of the deleted material.
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SubjectDeletion {
|
||||||
|
pub subject_id: String,
|
||||||
|
pub operation_id: String,
|
||||||
|
pub completed: bool,
|
||||||
|
pub started_at: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host-only cleanup identity, never Job input, output, profile or authorization material.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct DeletionSubjectJob {
|
||||||
|
pub job_id: String,
|
||||||
|
pub attempt_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// All content tables are enumerated deliberately. Indirect tables are erased before their
|
||||||
|
// owner rows, both for FK checks and so their fencing predicates remain resolvable.
|
||||||
|
const INDIRECT: &[(&str, &str)] = &[
|
||||||
|
(
|
||||||
|
"journal_generation_input_deliveries",
|
||||||
|
"generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"legacy_record_archive",
|
||||||
|
"json_extract(row_json, '$.subject_id')={subject}",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"journal_generation_deliveries",
|
||||||
|
"generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"journal_operations",
|
||||||
|
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"journal_generations",
|
||||||
|
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"journal_sources",
|
||||||
|
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"journal_session_bindings",
|
||||||
|
"binding_id IN (SELECT binding_id FROM journal_writer_bindings WHERE subject_id={subject})",
|
||||||
|
),
|
||||||
|
];
|
||||||
|
const DIRECT: &[&str] = &[
|
||||||
|
"memory_journal_links",
|
||||||
|
"journal_corrections",
|
||||||
|
"journal_entries",
|
||||||
|
"journals",
|
||||||
|
"journal_writer_bindings",
|
||||||
|
"subject_session_attributions",
|
||||||
|
"worker_execution_grants",
|
||||||
|
"subject_permission_receipts",
|
||||||
|
"subject_permissions",
|
||||||
|
"subject_permission_changes",
|
||||||
|
"candidate_decision_receipts",
|
||||||
|
"surface_generation_state",
|
||||||
|
"surface_generation_runs",
|
||||||
|
"surface_snapshot_refs",
|
||||||
|
"surface_snapshot_seals",
|
||||||
|
"surface_snapshots",
|
||||||
|
"staging_resolution_targets",
|
||||||
|
"staging_resolution_seals",
|
||||||
|
"staging_resolutions",
|
||||||
|
"memory_change_candidates",
|
||||||
|
"memory_change_derivations",
|
||||||
|
"memory_change_seals",
|
||||||
|
"memory_changes",
|
||||||
|
"memory_records",
|
||||||
|
"staging_records",
|
||||||
|
"subjects",
|
||||||
|
];
|
||||||
|
|
||||||
|
fn owner_expression(table: &str, row: &str) -> String {
|
||||||
|
match table {
|
||||||
|
"legacy_record_archive" => format!("json_extract({row}.row_json, '$.subject_id')"),
|
||||||
|
"journal_generation_input_deliveries" | "journal_generation_deliveries" => format!(
|
||||||
|
"(SELECT j.subject_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE g.id={row}.generation_id)"
|
||||||
|
),
|
||||||
|
"journal_operations" | "journal_generations" | "journal_sources" => {
|
||||||
|
format!("(SELECT subject_id FROM journals WHERE id={row}.journal_id)")
|
||||||
|
}
|
||||||
|
"journal_session_bindings" => format!(
|
||||||
|
"(SELECT subject_id FROM journal_writer_bindings WHERE binding_id={row}.binding_id)"
|
||||||
|
),
|
||||||
|
_ => format!("{row}.subject_id"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn migrate(tx: &Transaction<'_>) -> feature_storage::Result<()> {
|
||||||
|
tx.execute_batch(
|
||||||
|
"CREATE TABLE subject_deletions (
|
||||||
|
subject_id TEXT PRIMARY KEY,
|
||||||
|
operation_id TEXT NOT NULL UNIQUE,
|
||||||
|
completed INTEGER NOT NULL CHECK(completed IN (0,1)),
|
||||||
|
started_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE subject_deleted_workers (
|
||||||
|
subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT,
|
||||||
|
runtime_id TEXT NOT NULL,
|
||||||
|
worker_id TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(subject_id,runtime_id,worker_id)
|
||||||
|
);
|
||||||
|
CREATE TABLE subject_deleted_sessions (
|
||||||
|
subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT,
|
||||||
|
session_id TEXT NOT NULL UNIQUE,
|
||||||
|
PRIMARY KEY(subject_id,session_id)
|
||||||
|
);
|
||||||
|
CREATE TRIGGER subject_deleted_sessions_retained BEFORE DELETE ON subject_deleted_sessions
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'deleted Session fences are permanent'); END;
|
||||||
|
CREATE TRIGGER subject_deleted_sessions_immutable BEFORE UPDATE ON subject_deleted_sessions
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'deleted Session fences are immutable'); END;
|
||||||
|
CREATE INDEX subject_deleted_worker_identity ON subject_deleted_workers(runtime_id,worker_id);
|
||||||
|
CREATE TRIGGER subject_deletions_retained BEFORE DELETE ON subject_deletions
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'Subject deletion fences are permanent'); END;
|
||||||
|
CREATE TRIGGER subject_deletions_identity BEFORE UPDATE ON subject_deletions
|
||||||
|
WHEN OLD.subject_id<>NEW.subject_id OR OLD.operation_id<>NEW.operation_id
|
||||||
|
OR OLD.started_at<>NEW.started_at OR OLD.completed>NEW.completed
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'Subject deletion identity is permanent'); END;
|
||||||
|
CREATE TRIGGER subject_deleted_workers_retained BEFORE DELETE ON subject_deleted_workers
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are permanent'); END;
|
||||||
|
CREATE TRIGGER subject_deleted_workers_immutable BEFORE UPDATE ON subject_deleted_workers
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are immutable'); END;",
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Preserve each existing history-retention condition, adding only the explicit Subject
|
||||||
|
// erasure exception. Frozen earlier migrations are not edited. Normal history deletion
|
||||||
|
// (including another Subject's history) remains forbidden.
|
||||||
|
let triggers = tx
|
||||||
|
.prepare("SELECT name,tbl_name,sql FROM sqlite_master WHERE type='trigger'")?
|
||||||
|
.query_map([], |r| {
|
||||||
|
Ok((
|
||||||
|
r.get::<_, String>(0)?,
|
||||||
|
r.get::<_, String>(1)?,
|
||||||
|
r.get::<_, String>(2)?,
|
||||||
|
))
|
||||||
|
})?
|
||||||
|
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||||
|
for (name, table, sql) in triggers {
|
||||||
|
if !DIRECT.contains(&table.as_str()) && !INDIRECT.iter().any(|(t, _)| *t == table) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let upper = sql.to_ascii_uppercase();
|
||||||
|
if !upper.contains("BEFORE DELETE") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let begin = upper.find("BEGIN").expect("trigger body");
|
||||||
|
let (header, body) = sql.split_at(begin);
|
||||||
|
let exception = format!(
|
||||||
|
"NOT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id={})",
|
||||||
|
owner_expression(&table, "OLD")
|
||||||
|
);
|
||||||
|
let header = if let Some(when) = header.to_ascii_uppercase().find("WHEN") {
|
||||||
|
format!(
|
||||||
|
"{} WHEN ({}) AND {exception} ",
|
||||||
|
&header[..when],
|
||||||
|
header[when + 4..].trim()
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!("{header} WHEN {exception} ")
|
||||||
|
};
|
||||||
|
tx.execute_batch(&format!("DROP TRIGGER \"{name}\"; {header}{body}"))?;
|
||||||
|
}
|
||||||
|
for table in DIRECT
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.chain(INDIRECT.iter().map(|(t, _)| *t))
|
||||||
|
.chain(["journal_erased_sessions"])
|
||||||
|
{
|
||||||
|
for action in ["INSERT", "UPDATE"] {
|
||||||
|
let new_owner = owner_expression(table, "NEW");
|
||||||
|
let predicate = if action == "UPDATE" {
|
||||||
|
format!(
|
||||||
|
"subject_id={new_owner} OR subject_id={}",
|
||||||
|
owner_expression(table, "OLD")
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!("subject_id={new_owner}")
|
||||||
|
};
|
||||||
|
tx.execute_batch(&format!(
|
||||||
|
"CREATE TRIGGER deletion_fence_{table}_{action} BEFORE {action} ON {table}
|
||||||
|
WHEN EXISTS(SELECT 1 FROM subject_deletions WHERE {predicate})
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'Subject deletion has begun'); END;"
|
||||||
|
))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Identity reuse must not reconnect purged content or turn a deleted child into
|
||||||
|
// an unscoped Worker. Check both old and new on ownership-changing UPDATEs.
|
||||||
|
for table in [
|
||||||
|
"subject_session_attributions",
|
||||||
|
"journal_writer_bindings",
|
||||||
|
"worker_execution_grants",
|
||||||
|
] {
|
||||||
|
for action in ["INSERT", "UPDATE"] {
|
||||||
|
let rows = if action == "UPDATE" {
|
||||||
|
vec!["OLD", "NEW"]
|
||||||
|
} else {
|
||||||
|
vec!["NEW"]
|
||||||
|
};
|
||||||
|
let mut predicates = Vec::new();
|
||||||
|
for row in rows {
|
||||||
|
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id={row}.runtime_id AND worker_id={row}.worker_id)"));
|
||||||
|
if table == "subject_session_attributions" {
|
||||||
|
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_sessions WHERE session_id={row}.session_id)"));
|
||||||
|
}
|
||||||
|
if table == "worker_execution_grants" {
|
||||||
|
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=json_extract({row}.record_json,'$.origin_runtime_id') AND worker_id=json_extract({row}.record_json,'$.origin_worker_id'))"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tx.execute_batch(&format!("CREATE TRIGGER deletion_identity_{table}_{action} BEFORE {action} ON {table} WHEN {} BEGIN SELECT RAISE(ABORT, 'deleted Subject execution identity'); END;", predicates.join(" OR ")))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn is_deleting(connection: &Connection, subject: &str) -> Result<bool> {
|
||||||
|
// Older migration fixtures deliberately use current Rust reads before upgrade. Production
|
||||||
|
// opens only registered current stores and must fail closed if the fence table is absent.
|
||||||
|
#[cfg(test)]
|
||||||
|
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions' AND type='table')", [], |r|r.get::<_,bool>(0))? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
Ok(connection.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id=?1)",
|
||||||
|
[subject],
|
||||||
|
|r| r.get::<_, bool>(0),
|
||||||
|
)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn require_not_deleting(connection: &Connection, subject: &str) -> Result<()> {
|
||||||
|
if is_deleting(connection, subject)? {
|
||||||
|
return Err(SubjektivError::SubjectNotFound(subject.into()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
pub(crate) fn require_live_worker(
|
||||||
|
connection: &Connection,
|
||||||
|
runtime: &str,
|
||||||
|
worker: &str,
|
||||||
|
) -> Result<()> {
|
||||||
|
#[cfg(test)]
|
||||||
|
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_workers' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(()); }
|
||||||
|
if connection.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)",
|
||||||
|
params![runtime, worker],
|
||||||
|
|r| r.get::<_, bool>(0),
|
||||||
|
)? {
|
||||||
|
return Err(SubjektivError::AuthorizationDenied(
|
||||||
|
"Worker belongs to a deleted Subject".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn deleted_session_subject(
|
||||||
|
connection: &Connection,
|
||||||
|
session: &str,
|
||||||
|
) -> Result<Option<String>> {
|
||||||
|
#[cfg(test)]
|
||||||
|
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_sessions' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(None); }
|
||||||
|
Ok(connection
|
||||||
|
.query_row(
|
||||||
|
"SELECT subject_id FROM subject_deleted_sessions WHERE session_id=?1",
|
||||||
|
[session],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.optional()?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deletion(connection: &Connection, subject: &str) -> Result<Option<SubjectDeletion>> {
|
||||||
|
Ok(connection.query_row("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE subject_id=?1", [subject], |r|Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? })).optional()?)
|
||||||
|
}
|
||||||
|
fn require_deletion(connection: &Connection, subject: &str) -> Result<SubjectDeletion> {
|
||||||
|
deletion(connection, subject)?.ok_or_else(|| SubjektivError::SubjectNotFound(subject.into()))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SubjektivStore {
|
||||||
|
/// Commits the read/write/authority fence, even for legacy-retired Subjects. Operation IDs
|
||||||
|
/// are unique within this store's scope; neither another key nor another Subject may reuse
|
||||||
|
/// a deletion. Exact retries return the saved current result, including completed deletions.
|
||||||
|
pub fn begin_subject_deletion(
|
||||||
|
&self,
|
||||||
|
subject: &str,
|
||||||
|
operation: &str,
|
||||||
|
) -> Result<SubjectDeletion> {
|
||||||
|
validate_label("subject id", subject)?;
|
||||||
|
validate_label("deletion operation id", operation)?;
|
||||||
|
self.database.try_transaction(|tx| {
|
||||||
|
if let Some(saved) = deletion(tx,subject)? {
|
||||||
|
return if saved.operation_id == operation { Ok(saved) } else { Err(SubjektivError::SubjectDeletionConflict(operation.into())) };
|
||||||
|
}
|
||||||
|
if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE operation_id=?1)",[operation],|r|r.get::<_,bool>(0))? {
|
||||||
|
return Err(SubjektivError::SubjectDeletionConflict(operation.into()));
|
||||||
|
}
|
||||||
|
require_subject(tx,subject)?;
|
||||||
|
tx.execute("INSERT INTO subject_deletions(subject_id,operation_id,completed,started_at) VALUES(?1,?2,0,?3)",params![subject,operation,now()])?;
|
||||||
|
// Capture bodies, historical attribution, delegated children, and grant origins.
|
||||||
|
// This transaction also fences delayed grants/attributions, so no new identity can
|
||||||
|
// appear after the inventory is frozen. Runtime create reservations remain Host-owned.
|
||||||
|
tx.execute("INSERT INTO subject_deleted_workers(subject_id,runtime_id,worker_id)
|
||||||
|
SELECT subject_id,runtime_id,worker_id FROM subject_session_attributions WHERE subject_id=?1
|
||||||
|
UNION SELECT subject_id,runtime_id,worker_id FROM journal_writer_bindings WHERE subject_id=?1
|
||||||
|
UNION SELECT subject_id,runtime_id,worker_id FROM worker_execution_grants WHERE subject_id=?1
|
||||||
|
UNION SELECT subject_id,json_extract(record_json,'$.origin_runtime_id'),json_extract(record_json,'$.origin_worker_id') FROM worker_execution_grants WHERE subject_id=?1",[subject])?;
|
||||||
|
tx.execute("INSERT INTO subject_deleted_sessions(subject_id,session_id)
|
||||||
|
SELECT subject_id,session_id FROM subject_session_attributions WHERE subject_id=?1
|
||||||
|
UNION SELECT subject_id,source_session_id FROM journals WHERE subject_id=?1
|
||||||
|
UNION SELECT subject_id,session_id FROM journal_erased_sessions WHERE subject_id=?1", [subject])?;
|
||||||
|
require_deletion(tx,subject)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host-only, content-free cleanup inventory. Usable after begin, including exact finish
|
||||||
|
/// retries. Tombstones survive finish to prevent non-Subject permission fallback.
|
||||||
|
pub fn deletion_subject_workers(&self, subject: &str) -> Result<Vec<(String, String)>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
require_deletion(tx,subject)?;
|
||||||
|
Ok(tx.prepare("SELECT runtime_id,worker_id FROM subject_deleted_workers WHERE subject_id=?1 ORDER BY runtime_id,worker_id")?.query_map([subject],|r|Ok((r.get(0)?,r.get(1)?)))?.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host-only Session identities; retained source rows are never exposed through this path.
|
||||||
|
/// The Host must erase originals/archives before finish removes attribution.
|
||||||
|
pub fn deletion_subject_sessions(&self, subject: &str) -> Result<Vec<String>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
require_deletion(tx,subject)?;
|
||||||
|
Ok(tx.prepare("SELECT session_id FROM subject_deleted_sessions WHERE subject_id=?1 ORDER BY session_id")?.query_map([subject],|r|r.get(0))?.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host-only Job cleanup inventory. Job storage owns inputs, outcomes and Worker archives;
|
||||||
|
/// this store retains only these identities until Host cleanup has succeeded.
|
||||||
|
pub fn deletion_subject_jobs(&self, subject: &str) -> Result<Vec<DeletionSubjectJob>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
require_deletion(tx,subject)?;
|
||||||
|
Ok(tx.prepare("SELECT job_id,attempt_id FROM surface_generation_runs WHERE subject_id=?1 AND job_id IS NOT NULL AND attempt_id IS NOT NULL UNION SELECT g.job_id,g.attempt_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id=?1 ORDER BY 1,2")?.query_map([subject],|r|Ok(DeletionSubjectJob {job_id:r.get(0)?,attempt_id:r.get(1)?}))?.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host-only recovery state; this never exposes Subject content or authority.
|
||||||
|
pub fn subject_deletion(&self, subject: &str) -> Result<Option<SubjectDeletion>> {
|
||||||
|
self.database
|
||||||
|
.try_with_connection(|tx| deletion(tx, subject))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Durable unfinished operations, including after Host restart. Completed tombstones
|
||||||
|
/// remain queryable individually but are not returned as pending work.
|
||||||
|
pub fn pending_subject_deletions(&self) -> Result<Vec<SubjectDeletion>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
Ok(tx.prepare("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE completed=0 ORDER BY started_at,subject_id")?
|
||||||
|
.query_map([], |r| Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? }))?
|
||||||
|
.collect::<std::result::Result<Vec<_>,_>>()?)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn worker_subject_deleted(&self, runtime: &str, worker: &str) -> Result<bool> {
|
||||||
|
self.database.try_with_connection(|tx| Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)",params![runtime,worker],|r|r.get(0))?))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host confirmation that external cleanup is complete. All Subject content and completed
|
||||||
|
/// receipt commit together. Deferred FKs allow cyclic current/history references to disappear
|
||||||
|
/// without weakening FK enforcement outside this transaction. Failure rolls back all erasure.
|
||||||
|
pub fn finish_subject_deletion(
|
||||||
|
&self,
|
||||||
|
subject: &str,
|
||||||
|
operation: &str,
|
||||||
|
) -> Result<SubjectDeletion> {
|
||||||
|
self.database.try_transaction(|tx| {
|
||||||
|
let saved = require_deletion(tx, subject)?;
|
||||||
|
if saved.operation_id != operation {
|
||||||
|
return Err(SubjektivError::SubjectDeletionConflict(operation.into()));
|
||||||
|
}
|
||||||
|
if saved.completed {
|
||||||
|
return Ok(saved);
|
||||||
|
}
|
||||||
|
tx.execute_batch("PRAGMA defer_foreign_keys=ON;")?;
|
||||||
|
for (table, predicate) in INDIRECT {
|
||||||
|
tx.execute(
|
||||||
|
&format!(
|
||||||
|
"DELETE FROM {table} WHERE {}",
|
||||||
|
predicate.replace("{subject}", "?1")
|
||||||
|
),
|
||||||
|
[subject],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
for table in DIRECT {
|
||||||
|
tx.execute(
|
||||||
|
&format!("DELETE FROM {table} WHERE subject_id=?1"),
|
||||||
|
[subject],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE subject_deletions SET completed=1 WHERE subject_id=?1",
|
||||||
|
[subject],
|
||||||
|
)?;
|
||||||
|
require_deletion(tx, subject)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,489 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::tests::{candidate, legacy_retired_fixture, open_store, role};
|
||||||
|
use feature_storage::FeatureStorage;
|
||||||
|
use memory::extract::CandidateKind;
|
||||||
|
|
||||||
|
fn draft() -> MemoryDraft {
|
||||||
|
MemoryDraft::active(
|
||||||
|
CandidateKind::Lesson,
|
||||||
|
"private claim",
|
||||||
|
"private body",
|
||||||
|
"useful",
|
||||||
|
"initial",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count(store: &SubjektivStore, table: &str) -> i64 {
|
||||||
|
store
|
||||||
|
.database
|
||||||
|
.try_with_connection::<_, SubjektivError>(|c| {
|
||||||
|
Ok(c.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0))?)
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fence_only_denies_content_reads_writes_and_exact_replay_before_purge() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
let subject = store
|
||||||
|
.create_subject_with_behavior(role(), "private behavior".into())
|
||||||
|
.unwrap();
|
||||||
|
let id = &subject.id;
|
||||||
|
let attribution = SubjectSessionAttribution::new(id, "runtime", "worker", "session").unwrap();
|
||||||
|
store
|
||||||
|
.record_session_attribution(attribution.clone())
|
||||||
|
.unwrap();
|
||||||
|
let staged = candidate(id, "candidate", store.workspace_id());
|
||||||
|
store.stage_candidate(staged.clone()).unwrap();
|
||||||
|
let request = CandidateDecisionRequest {
|
||||||
|
request_id: "decision".into(),
|
||||||
|
candidate_id: "candidate".into(),
|
||||||
|
reason: "adopt".into(),
|
||||||
|
decision: CandidateDecision::Apply {
|
||||||
|
target: MemoryChangeTarget::Create,
|
||||||
|
draft: draft(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let memory = store
|
||||||
|
.decide_candidate(id, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.memory
|
||||||
|
.unwrap();
|
||||||
|
let generation = store
|
||||||
|
.prepare_job_surface_generation(id, Some(("job", "attempt")))
|
||||||
|
.unwrap();
|
||||||
|
let snapshot = store
|
||||||
|
.publish_surface_generation(
|
||||||
|
id,
|
||||||
|
&generation.id,
|
||||||
|
vec![SurfacePoint {
|
||||||
|
body_md: "private surface".into(),
|
||||||
|
memory_refs: vec![MemoryChangeRef {
|
||||||
|
memory_id: memory.id.clone(),
|
||||||
|
change_id: memory.change_id.clone(),
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let fence = store.begin_subject_deletion(id, "delete").unwrap();
|
||||||
|
assert!(!fence.completed);
|
||||||
|
// The fence must work while every body remains physically present.
|
||||||
|
assert_eq!(count(&store, "subjects"), 1);
|
||||||
|
assert_eq!(count(&store, "memory_records"), 1);
|
||||||
|
assert_eq!(count(&store, "candidate_decision_receipts"), 1);
|
||||||
|
assert!(store.subject(id).unwrap().is_none());
|
||||||
|
assert!(store.list_subjects(1).unwrap().items.is_empty());
|
||||||
|
macro_rules! denied { ($($result:expr),+ $(,)?) => { $(assert!($result.is_err(), stringify!($result));)+ }; }
|
||||||
|
denied!(
|
||||||
|
store.session_attribution("session"),
|
||||||
|
store.subject_session_attributions(id),
|
||||||
|
store.subject_session_attribution_page(id, None, "2999", None, 1),
|
||||||
|
store.record_session_attribution(attribution),
|
||||||
|
store.staging_candidate(id, "candidate"),
|
||||||
|
store.pending_staging_candidates(id, 1),
|
||||||
|
store.pending_staging_backlog(id),
|
||||||
|
store.staging_resolution(id, "candidate"),
|
||||||
|
store.memory(id, &memory.id),
|
||||||
|
store.memory_change(id, &memory.id, memory.change_id.clone()),
|
||||||
|
store.scoped_memory(id, &memory.id),
|
||||||
|
store.scoped_memory_change(id, &memory.id, memory.change_id.clone()),
|
||||||
|
store.list_memories(id),
|
||||||
|
store.list_memory_changes(id, &memory.id),
|
||||||
|
store.memory_fingerprint(id),
|
||||||
|
store.surface_snapshot(id, &snapshot.id),
|
||||||
|
store.resident_surface(id),
|
||||||
|
store.resident_context(id),
|
||||||
|
store.require_job_surface_generation(id, &generation.id, "job", "attempt"),
|
||||||
|
store.validate_job_surface_outcome(
|
||||||
|
id,
|
||||||
|
&generation.id,
|
||||||
|
generation.memory_fingerprint,
|
||||||
|
"ready",
|
||||||
|
Some(&snapshot.id),
|
||||||
|
None,
|
||||||
|
"job",
|
||||||
|
"attempt"
|
||||||
|
),
|
||||||
|
store.prepare_surface_generation(id),
|
||||||
|
store.publish_surface_generation(id, &generation.id, vec![]),
|
||||||
|
store.fail_surface_generation(id, &generation.id, "failed"),
|
||||||
|
store.update_subject_behavior(id, &subject.behavior_md, "late behavior".into()),
|
||||||
|
store.stage_candidate(staged),
|
||||||
|
store.decide_candidate(id, request),
|
||||||
|
store.create_memory(id, draft()),
|
||||||
|
store.subject_permissions(id),
|
||||||
|
store.list_subject_worker_execution_grants(id),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.deletion_subject_workers(id).unwrap(),
|
||||||
|
vec![("runtime".into(), "worker".into())]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.deletion_subject_sessions(id).unwrap(),
|
||||||
|
vec!["session"]
|
||||||
|
);
|
||||||
|
assert_eq!(store.deletion_subject_jobs(id).unwrap()[0].job_id, "job");
|
||||||
|
assert!(store.worker_subject_deleted("runtime", "worker").unwrap());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.finish_subject_deletion(id, "delete")
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
for table in DIRECT {
|
||||||
|
assert_eq!(count(&store, table), 0, "{table}");
|
||||||
|
}
|
||||||
|
assert!(store.worker_execution_grant("runtime", "worker").is_err());
|
||||||
|
assert_eq!(
|
||||||
|
store.deletion_subject_sessions(id).unwrap(),
|
||||||
|
vec!["session"]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.finish_subject_deletion(id, "delete").unwrap(),
|
||||||
|
store.begin_subject_deletion(id, "delete").unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn purge_failure_rolls_back_content_but_not_the_committed_fence_and_restart_recovers() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let id: String;
|
||||||
|
let pending;
|
||||||
|
{
|
||||||
|
let (manager, scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
id = store.create_subject(role()).unwrap().id;
|
||||||
|
store.create_memory(&id, draft()).unwrap();
|
||||||
|
pending = store.begin_subject_deletion(&id, "delete").unwrap();
|
||||||
|
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||||
|
c.execute_batch("CREATE TRIGGER injected_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT, 'simulated purge failure'); END;")?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert!(store.finish_subject_deletion(&id, "delete").is_err());
|
||||||
|
assert_eq!(count(&store, "memory_records"), 1);
|
||||||
|
assert_eq!(count(&store, "memory_changes"), 1);
|
||||||
|
assert_eq!(store.subject_deletion(&id).unwrap(), Some(pending.clone()));
|
||||||
|
drop(store);
|
||||||
|
drop(scope);
|
||||||
|
manager.shutdown().unwrap();
|
||||||
|
}
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
assert_eq!(
|
||||||
|
store.pending_subject_deletions().unwrap(),
|
||||||
|
vec![pending.clone()]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.begin_subject_deletion(&id, "delete").unwrap(),
|
||||||
|
pending
|
||||||
|
);
|
||||||
|
assert!(store.list_memories(&id).is_err());
|
||||||
|
store
|
||||||
|
.database
|
||||||
|
.try_with_connection::<_, SubjektivError>(|c| {
|
||||||
|
c.execute_batch("DROP TRIGGER injected_failure")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.finish_subject_deletion(&id, "delete")
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
assert!(store.pending_subject_deletions().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deletion_identity_conflicts_are_scope_wide_but_do_not_cross_workspaces() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
let a = store.create_subject(role()).unwrap().id;
|
||||||
|
let b = store.create_subject(role()).unwrap().id;
|
||||||
|
let saved = store.begin_subject_deletion(&a, "same-key").unwrap();
|
||||||
|
assert_eq!(store.begin_subject_deletion(&a, "same-key").unwrap(), saved);
|
||||||
|
for result in [
|
||||||
|
store.begin_subject_deletion(&a, "different"),
|
||||||
|
store.begin_subject_deletion(&b, "same-key"),
|
||||||
|
store.finish_subject_deletion(&a, "different"),
|
||||||
|
] {
|
||||||
|
assert!(matches!(
|
||||||
|
result,
|
||||||
|
Err(SubjektivError::SubjectDeletionConflict(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
assert!(store.begin_subject_deletion("absent", "new-key").is_err());
|
||||||
|
assert!(store.finish_subject_deletion(&b, "same-key").is_err());
|
||||||
|
let page = store.list_subjects(1).unwrap();
|
||||||
|
assert_eq!(page.items[0].id, b);
|
||||||
|
assert!(!page.has_more);
|
||||||
|
store.finish_subject_deletion(&a, "same-key").unwrap();
|
||||||
|
store.create_memory(&b, draft()).unwrap();
|
||||||
|
let (_other_manager, _other_scope, other) = open_store(temp.path(), "workspace-b");
|
||||||
|
assert!(other.subject_deletion(&a).unwrap().is_none());
|
||||||
|
assert!(other.begin_subject_deletion(&b, "same-key").is_err());
|
||||||
|
let c = other.create_subject(role()).unwrap().id;
|
||||||
|
other.begin_subject_deletion(&c, "same-key").unwrap();
|
||||||
|
assert_eq!(store.list_memories(&b).unwrap().len(), 1);
|
||||||
|
let fresh = store.create_subject(role()).unwrap().id;
|
||||||
|
assert!(store.list_memories(&fresh).unwrap().is_empty());
|
||||||
|
assert!(store.subject_permissions(&fresh).unwrap().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn concurrent_begin_and_write_serialize_and_exact_finish_is_replayable() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
let id = store.create_subject(role()).unwrap().id;
|
||||||
|
let barrier = std::sync::Barrier::new(3);
|
||||||
|
std::thread::scope(|scope| {
|
||||||
|
let a = scope.spawn(|| {
|
||||||
|
barrier.wait();
|
||||||
|
store.begin_subject_deletion(&id, "delete")
|
||||||
|
});
|
||||||
|
let b = scope.spawn(|| {
|
||||||
|
barrier.wait();
|
||||||
|
store.create_memory(&id, draft())
|
||||||
|
});
|
||||||
|
barrier.wait();
|
||||||
|
assert!(!a.join().unwrap().unwrap().completed);
|
||||||
|
// A write either precedes begin and is later purged, or loses to the fence.
|
||||||
|
let _ = b.join().unwrap();
|
||||||
|
});
|
||||||
|
assert!(store.create_memory(&id, draft()).is_err());
|
||||||
|
assert_eq!(store.pending_subject_deletions().unwrap().len(), 1);
|
||||||
|
store.finish_subject_deletion(&id, "delete").unwrap();
|
||||||
|
assert_eq!(count(&store, "memory_records"), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_subject_owned_table_has_old_and_new_sql_fences_and_retention_is_scoped() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
let a = store.create_subject(role()).unwrap().id;
|
||||||
|
let b = store.create_subject(role()).unwrap().id;
|
||||||
|
store
|
||||||
|
.stage_candidate(candidate(&a, "a", store.workspace_id()))
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.stage_candidate(candidate(&b, "b", store.workspace_id()))
|
||||||
|
.unwrap();
|
||||||
|
store.begin_subject_deletion(&a, "delete").unwrap();
|
||||||
|
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||||
|
let tables = c.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")?.query_map([], |r| r.get::<_,String>(0))?.collect::<std::result::Result<Vec<_>,_>>()?;
|
||||||
|
for table in tables {
|
||||||
|
let columns = c.prepare(&format!("PRAGMA table_info({table})"))?.query_map([], |r| r.get::<_,String>(1))?.collect::<std::result::Result<Vec<_>,_>>()?;
|
||||||
|
if !columns.iter().any(|v| v == "subject_id") || table.starts_with("subject_delet") { continue; }
|
||||||
|
assert!(DIRECT.contains(&table.as_str()) || table == "journal_erased_sessions", "unclassified Subject table: {table}");
|
||||||
|
}
|
||||||
|
for table in DIRECT.iter().copied().chain(INDIRECT.iter().map(|v| v.0)).chain(["journal_erased_sessions"]) {
|
||||||
|
let sql: String = c.query_row("SELECT sql FROM sqlite_master WHERE name=?1", [format!("deletion_fence_{table}_UPDATE")], |r| r.get(0))?;
|
||||||
|
assert!(sql.contains("OLD.") && sql.contains("NEW."), "{table}: {sql}");
|
||||||
|
}
|
||||||
|
// OLD fenced -> NEW unfenced must not evade a guard.
|
||||||
|
assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![a, "new-id"]).is_err());
|
||||||
|
assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![b, a]).is_err());
|
||||||
|
assert!(c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&b]).is_err());
|
||||||
|
assert!(c.execute("DELETE FROM subject_deletions WHERE subject_id=?1", [&a]).is_err());
|
||||||
|
assert!(c.execute("UPDATE subject_deletions SET operation_id='changed' WHERE subject_id=?1", [&a]).is_err());
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
store.finish_subject_deletion(&a, "delete").unwrap();
|
||||||
|
assert!(store.staging_candidate(&b, "b").unwrap().is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_retired_is_readable_serde_compatible_and_explicitly_deletable() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
let id = store.create_subject(role()).unwrap().id;
|
||||||
|
store.create_memory(&id, draft()).unwrap();
|
||||||
|
legacy_retired_fixture(&store, &id).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<SubjectState>("\"retired\"").unwrap(),
|
||||||
|
SubjectState::LegacyRetired
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&SubjectState::LegacyRetired).unwrap(),
|
||||||
|
"\"retired\""
|
||||||
|
);
|
||||||
|
assert_eq!(store.list_memories(&id).unwrap().len(), 1);
|
||||||
|
assert!(store.create_memory(&id, draft()).is_err());
|
||||||
|
store.begin_subject_deletion(&id, "delete-legacy").unwrap();
|
||||||
|
store.finish_subject_deletion(&id, "delete-legacy").unwrap();
|
||||||
|
assert_eq!(count(&store, "memory_changes"), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn migration_preserves_legacy_content_until_explicit_delete_including_archive() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let id: String;
|
||||||
|
{
|
||||||
|
let manager = FeatureStorage::new(temp.path());
|
||||||
|
let scope = manager.workspace("workspace-a").unwrap();
|
||||||
|
let registration = scope
|
||||||
|
.register(FeatureRegistration::new(
|
||||||
|
SUBJEKTIV_FEATURE_ID,
|
||||||
|
&MIGRATIONS[..6],
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||||
|
id = "legacy-subject".into();
|
||||||
|
store.database.try_transaction::<_, SubjektivError>(|tx| {
|
||||||
|
let raw = serde_json::json!({"schema_version":1,"id":id,"role":"workspace_companion","behavior_md":"legacy private behavior","state":"active","created_at":"2026-01-01","updated_at":"2026-01-01"}).to_string();
|
||||||
|
tx.execute("INSERT INTO subjects(subject_id,role,state,store_revision,record_json,created_at,updated_at) VALUES(?1,'workspace_companion','active',0,?2,'2026-01-01','2026-01-01')", params![id,raw])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
// Build a saved SQL fixture rather than using current-only APIs against an old schema.
|
||||||
|
store
|
||||||
|
.database
|
||||||
|
.try_transaction::<_, SubjektivError>(|tx| {
|
||||||
|
let raw: String = tx.query_row(
|
||||||
|
"SELECT record_json FROM subjects WHERE subject_id=?1",
|
||||||
|
[&id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
let mut value: serde_json::Value = serde_json::from_str(&raw)?;
|
||||||
|
value["state"] = "retired".into();
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE subjects SET state='retired',record_json=?2 WHERE subject_id=?1",
|
||||||
|
params![id, value.to_string()],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
drop(store);
|
||||||
|
drop(scope);
|
||||||
|
manager.shutdown().unwrap();
|
||||||
|
}
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
assert_eq!(
|
||||||
|
store.subject(&id).unwrap().unwrap().state,
|
||||||
|
SubjectState::LegacyRetired
|
||||||
|
);
|
||||||
|
assert!(count(&store, "legacy_record_archive") > 1);
|
||||||
|
store.begin_subject_deletion(&id, "delete").unwrap();
|
||||||
|
store.finish_subject_deletion(&id, "delete").unwrap();
|
||||||
|
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||||
|
let remaining: i64 = c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE json_extract(row_json,'$.subject_id')=?1", [&id], |r|r.get(0))?;
|
||||||
|
assert_eq!(remaining, 0);
|
||||||
|
// Workspace-only metadata remains and is not a copy of the Subject's content.
|
||||||
|
assert_eq!(c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE source_table='store_scope'", [], |r|r.get::<_,i64>(0))?, 1);
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn concurrent_deletions_on_independent_connections_save_one_request_identity() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
let (_second_manager, _second_scope, second) = open_store(temp.path(), "workspace-a");
|
||||||
|
let id = store.create_subject(role()).unwrap().id;
|
||||||
|
let barrier = std::sync::Barrier::new(3);
|
||||||
|
std::thread::scope(|scope| {
|
||||||
|
let a = scope.spawn(|| {
|
||||||
|
barrier.wait();
|
||||||
|
store.begin_subject_deletion(&id, "first")
|
||||||
|
});
|
||||||
|
let b = scope.spawn(|| {
|
||||||
|
barrier.wait();
|
||||||
|
second.begin_subject_deletion(&id, "second")
|
||||||
|
});
|
||||||
|
barrier.wait();
|
||||||
|
let results = [a.join().unwrap(), b.join().unwrap()];
|
||||||
|
assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
results
|
||||||
|
.iter()
|
||||||
|
.filter(|r| matches!(r, Err(SubjektivError::SubjectDeletionConflict(_))))
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
});
|
||||||
|
let saved = second.subject_deletion(&id).unwrap().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.begin_subject_deletion(&id, &saved.operation_id)
|
||||||
|
.unwrap(),
|
||||||
|
saved
|
||||||
|
);
|
||||||
|
std::thread::scope(|scope| {
|
||||||
|
let a = scope.spawn(|| store.finish_subject_deletion(&id, &saved.operation_id));
|
||||||
|
let b = scope.spawn(|| second.finish_subject_deletion(&id, &saved.operation_id));
|
||||||
|
assert_eq!(a.join().unwrap().unwrap(), b.join().unwrap().unwrap());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deletion_migration_failure_rolls_back_fences_and_trigger_rewrites_then_retries() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let id;
|
||||||
|
{
|
||||||
|
let manager = FeatureStorage::new(temp.path());
|
||||||
|
let scope = manager.workspace("workspace-a").unwrap();
|
||||||
|
let registration = scope
|
||||||
|
.register(FeatureRegistration::new(
|
||||||
|
SUBJEKTIV_FEATURE_ID,
|
||||||
|
&MIGRATIONS[..13],
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||||
|
id = store.create_subject(role()).unwrap().id;
|
||||||
|
store
|
||||||
|
.stage_candidate(candidate(&id, "preserved", store.workspace_id()))
|
||||||
|
.unwrap();
|
||||||
|
store.database.try_with_connection::<_, SubjektivError>(|c| {
|
||||||
|
// Fail near the end, after retention triggers have already been rewritten.
|
||||||
|
c.execute_batch("CREATE TRIGGER deletion_fence_journal_sources_INSERT BEFORE INSERT ON journal_sources BEGIN SELECT 1; END;")?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
drop(store);
|
||||||
|
drop(scope);
|
||||||
|
manager.shutdown().unwrap();
|
||||||
|
}
|
||||||
|
let manager = FeatureStorage::new(temp.path());
|
||||||
|
let scope = manager.workspace("workspace-a").unwrap();
|
||||||
|
let current = SubjektivStore::register(&scope).unwrap();
|
||||||
|
assert!(SubjektivStore::open(&scope, ¤t).is_err());
|
||||||
|
drop(scope);
|
||||||
|
manager.shutdown().unwrap();
|
||||||
|
let manager = FeatureStorage::new(temp.path());
|
||||||
|
let scope = manager.workspace("workspace-a").unwrap();
|
||||||
|
// Reopen only the fixture's pre-deletion registration to inspect the failed migration.
|
||||||
|
let old = scope
|
||||||
|
.register(FeatureRegistration::new(
|
||||||
|
SUBJEKTIV_FEATURE_ID,
|
||||||
|
&MIGRATIONS[..13],
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
let store = SubjektivStore::open(&scope, &old).unwrap();
|
||||||
|
store
|
||||||
|
.database
|
||||||
|
.try_with_connection::<_, SubjektivError>(|c| {
|
||||||
|
assert!(!c.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions')",
|
||||||
|
[],
|
||||||
|
|r| r.get::<_, bool>(0)
|
||||||
|
)?);
|
||||||
|
let sql: String = c.query_row(
|
||||||
|
"SELECT sql FROM sqlite_master WHERE name='staging_records_no_delete'",
|
||||||
|
[],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
assert!(!sql.contains("subject_deletions"));
|
||||||
|
assert!(
|
||||||
|
c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&id])
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
c.execute_batch("DROP TRIGGER deletion_fence_journal_sources_INSERT;")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
drop(store);
|
||||||
|
drop(scope);
|
||||||
|
manager.shutdown().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
assert!(store.staging_candidate(&id, "preserved").unwrap().is_some());
|
||||||
|
assert!(store.pending_subject_deletions().unwrap().is_empty());
|
||||||
|
store.begin_subject_deletion(&id, "delete").unwrap();
|
||||||
|
store.finish_subject_deletion(&id, "delete").unwrap();
|
||||||
|
}
|
||||||
@@ -39,6 +39,7 @@ fn denied() -> JournalError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn check_writer(tx: &Connection, host: &JournalHostContext) -> JournalResult<()> {
|
pub(super) fn check_writer(tx: &Connection, host: &JournalHostContext) -> JournalResult<()> {
|
||||||
|
crate::require_subject_in_connection(tx, &host.subject_id)?;
|
||||||
let attr = host.attribution.as_ref().ok_or_else(denied)?;
|
let attr = host.attribution.as_ref().ok_or_else(denied)?;
|
||||||
let binding = host.writer_binding_id.as_ref().ok_or_else(denied)?;
|
let binding = host.writer_binding_id.as_ref().ok_or_else(denied)?;
|
||||||
if !tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings b JOIN journal_session_bindings s ON s.binding_id=b.binding_id WHERE b.binding_id=?1 AND b.subject_id=?2 AND b.runtime_id=?3 AND b.worker_id=?4 AND b.active=1 AND s.session_id=?5)", params![binding,host.subject_id,attr.runtime_id,attr.worker_id,attr.session_id], |r| r.get::<_,bool>(0))? {
|
if !tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings b JOIN journal_session_bindings s ON s.binding_id=b.binding_id WHERE b.binding_id=?1 AND b.subject_id=?2 AND b.runtime_id=?3 AND b.worker_id=?4 AND b.active=1 AND s.session_id=?5)", params![binding,host.subject_id,attr.runtime_id,attr.worker_id,attr.session_id], |r| r.get::<_,bool>(0))? {
|
||||||
@@ -63,6 +64,7 @@ impl SubjektivStore {
|
|||||||
binding: &JournalWriterBinding,
|
binding: &JournalWriterBinding,
|
||||||
) -> JournalResult<Option<String>> {
|
) -> JournalResult<Option<String>> {
|
||||||
self.database.try_with_connection(|tx| {
|
self.database.try_with_connection(|tx| {
|
||||||
|
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||||
Ok(tx.query_row("SELECT j.source_session_id FROM journals j JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE j.subject_id=?1 AND s.binding_id=(SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=0 ORDER BY rowid DESC LIMIT 1) ORDER BY j.updated_at DESC,j.id DESC LIMIT 1", [&binding.subject_id], |r| r.get(0)).optional()?)
|
Ok(tx.query_row("SELECT j.source_session_id FROM journals j JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE j.subject_id=?1 AND s.binding_id=(SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=0 ORDER BY rowid DESC LIMIT 1) ORDER BY j.updated_at DESC,j.id DESC LIMIT 1", [&binding.subject_id], |r| r.get(0)).optional()?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -77,6 +79,7 @@ impl SubjektivStore {
|
|||||||
worker: &str,
|
worker: &str,
|
||||||
) -> JournalResult<bool> {
|
) -> JournalResult<bool> {
|
||||||
self.database.try_with_connection(|tx| {
|
self.database.try_with_connection(|tx| {
|
||||||
|
crate::require_subject_in_connection(tx, subject)?;
|
||||||
Ok(tx.query_row(
|
Ok(tx.query_row(
|
||||||
"SELECT NOT EXISTS(SELECT 1 FROM journal_writer_bindings
|
"SELECT NOT EXISTS(SELECT 1 FROM journal_writer_bindings
|
||||||
WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)
|
WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)
|
||||||
@@ -95,12 +98,14 @@ impl SubjektivStore {
|
|||||||
worker: &str,
|
worker: &str,
|
||||||
) -> JournalResult<bool> {
|
) -> JournalResult<bool> {
|
||||||
self.database.try_with_connection(|tx| {
|
self.database.try_with_connection(|tx| {
|
||||||
|
crate::require_subject_in_connection(tx, subject)?;
|
||||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 AND active=0)", params![subject,runtime,worker], |r| r.get(0))?)
|
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 AND active=0)", params![subject,runtime,worker], |r| r.get(0))?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn journal_writer_active(&self, binding: &JournalWriterBinding) -> JournalResult<bool> {
|
pub fn journal_writer_active(&self, binding: &JournalWriterBinding) -> JournalResult<bool> {
|
||||||
self.database.try_with_connection(|tx| {
|
self.database.try_with_connection(|tx| {
|
||||||
|
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4 AND active=1)", params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id], |r| r.get(0))?)
|
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4 AND active=1)", params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id], |r| r.get(0))?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -111,6 +116,7 @@ impl SubjektivStore {
|
|||||||
session: &str,
|
session: &str,
|
||||||
) -> JournalResult<bool> {
|
) -> JournalResult<bool> {
|
||||||
self.database.try_with_connection(|tx| {
|
self.database.try_with_connection(|tx| {
|
||||||
|
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_session_bindings WHERE binding_id=?1 AND session_id=?2)", params![binding.binding_id,session], |r| r.get(0))?)
|
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_session_bindings WHERE binding_id=?1 AND session_id=?2)", params![binding.binding_id,session], |r| r.get(0))?)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -135,6 +141,8 @@ impl SubjektivStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.database.try_transaction(|tx| {
|
self.database.try_transaction(|tx| {
|
||||||
|
crate::require_subject_in_connection(tx, subject)?;
|
||||||
|
crate::deletion::require_live_worker(tx, runtime, worker)?;
|
||||||
let state: Option<String> = tx.query_row("SELECT state FROM subjects WHERE subject_id=?1", [subject], |r| r.get(0)).optional()?;
|
let state: Option<String> = tx.query_row("SELECT state FROM subjects WHERE subject_id=?1", [subject], |r| r.get(0)).optional()?;
|
||||||
if state.as_deref() != Some("active") { return Err(JournalError::PermissionDenied("Subject is absent or retired".into())); }
|
if state.as_deref() != Some("active") { return Err(JournalError::PermissionDenied("Subject is absent or retired".into())); }
|
||||||
let current: Option<JournalWriterBinding> = tx.query_row("SELECT subject_id,runtime_id,worker_id,binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=1", [subject], |r| Ok(JournalWriterBinding { subject_id:r.get(0)?,runtime_id:r.get(1)?,worker_id:r.get(2)?,binding_id:r.get(3)? })).optional()?;
|
let current: Option<JournalWriterBinding> = tx.query_row("SELECT subject_id,runtime_id,worker_id,binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=1", [subject], |r| Ok(JournalWriterBinding { subject_id:r.get(0)?,runtime_id:r.get(1)?,worker_id:r.get(2)?,binding_id:r.get(3)? })).optional()?;
|
||||||
@@ -200,6 +208,7 @@ impl SubjektivStore {
|
|||||||
/// It never fixes an open entry or declares the represented work complete.
|
/// It never fixes an open entry or declares the represented work complete.
|
||||||
pub fn release_journal_writer(&self, binding: &JournalWriterBinding) -> JournalResult<()> {
|
pub fn release_journal_writer(&self, binding: &JournalWriterBinding) -> JournalResult<()> {
|
||||||
self.database.try_transaction(|tx| {
|
self.database.try_transaction(|tx| {
|
||||||
|
crate::require_subject_in_connection(tx, &binding.subject_id)?;
|
||||||
let active: Option<bool> = tx.query_row("SELECT active FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0)).optional()?;
|
let active: Option<bool> = tx.query_row("SELECT active FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0)).optional()?;
|
||||||
let Some(active) = active else { return Err(denied()); };
|
let Some(active) = active else { return Err(denied()); };
|
||||||
if !active { return Ok(()); }
|
if !active { return Ok(()); }
|
||||||
|
|||||||
@@ -1158,3 +1158,69 @@ fn failed_link_migration_rolls_back_ddl_and_preserves_existing_data_for_retry()
|
|||||||
assert_eq!(linked(&f.links(&selector, true).sources[0]).1, entry);
|
assert_eq!(linked(&f.links(&selector, true).sources[0]).1, entry);
|
||||||
assert_eq!(f.snapshot(), before);
|
assert_eq!(f.snapshot(), before);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_deletion_denies_all_memory_journal_context_and_reconcile_entry_points() {
|
||||||
|
let f = Fixture::new();
|
||||||
|
let staged = f.stage("candidate", &[0]);
|
||||||
|
let selector = f.apply(&staged);
|
||||||
|
let entry = f.fixed("entry", 0, vec![]);
|
||||||
|
f.links(&selector, true);
|
||||||
|
f.store()
|
||||||
|
.begin_subject_deletion(&f.host.subject_id, "delete")
|
||||||
|
.unwrap();
|
||||||
|
for request in [
|
||||||
|
MemoryJournalRequest::Links {
|
||||||
|
selector: selector.clone(),
|
||||||
|
},
|
||||||
|
MemoryJournalRequest::Reconcile {
|
||||||
|
selector: selector.clone(),
|
||||||
|
},
|
||||||
|
MemoryJournalRequest::LinksPage {
|
||||||
|
selector: selector.clone(),
|
||||||
|
source_offset: 0,
|
||||||
|
limit: 1,
|
||||||
|
},
|
||||||
|
MemoryJournalRequest::ReconcilePage {
|
||||||
|
selector: selector.clone(),
|
||||||
|
source_offset: 0,
|
||||||
|
limit: 1,
|
||||||
|
},
|
||||||
|
MemoryJournalRequest::Read {
|
||||||
|
selector,
|
||||||
|
source_index: 0,
|
||||||
|
journal_id: f.journal_id.clone(),
|
||||||
|
entry_id: entry.clone(),
|
||||||
|
expected_token: None,
|
||||||
|
byte_offset: 0,
|
||||||
|
max_bytes: 8,
|
||||||
|
},
|
||||||
|
MemoryJournalRequest::Related {
|
||||||
|
journal_id: f.journal_id.clone(),
|
||||||
|
entry_id: entry,
|
||||||
|
after: None,
|
||||||
|
limit: 1,
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
assert!(matches!(
|
||||||
|
f.store().execute_memory_journal(&f.host, request),
|
||||||
|
Err(JournalError::Subject(
|
||||||
|
crate::SubjektivError::SubjectNotFound(_)
|
||||||
|
))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
f.store()
|
||||||
|
.finish_subject_deletion(&f.host.subject_id, "delete")
|
||||||
|
.unwrap();
|
||||||
|
f.store()
|
||||||
|
.database
|
||||||
|
.try_with_connection::<_, JournalError>(|c| {
|
||||||
|
assert_eq!(
|
||||||
|
c.query_row("SELECT COUNT(*) FROM memory_journal_links", [], |r| r
|
||||||
|
.get::<_, i64>(0))?,
|
||||||
|
0
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|||||||
@@ -63,6 +63,10 @@ impl JournalInput {
|
|||||||
// below operates on this value only, never by making another live capture request.
|
// below operates on this value only, never by making another live capture request.
|
||||||
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
|
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
|
||||||
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
|
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
|
||||||
|
// Capture runs outside SQLite; a delayed callback cannot deliver deleted content.
|
||||||
|
store
|
||||||
|
.database
|
||||||
|
.try_with_connection(|tx| super::store::check_host(store, tx, host, true))?;
|
||||||
input.selected_ranges = capture_ranges(&input.capture)?;
|
input.selected_ranges = capture_ranges(&input.capture)?;
|
||||||
if input.selected_ranges.is_empty() {
|
if input.selected_ranges.is_empty() {
|
||||||
return Ok(input);
|
return Ok(input);
|
||||||
|
|||||||
@@ -469,6 +469,11 @@ fn execute_inner(
|
|||||||
Ok::<_, JournalError>(())
|
Ok::<_, JournalError>(())
|
||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
|
// Also check empty source reads and Initial snapshot responses (no delivery write).
|
||||||
|
store.database.try_with_connection(|tx| {
|
||||||
|
super::store::job(store, tx, context, false)?;
|
||||||
|
Ok::<_, JournalError>(())
|
||||||
|
})?;
|
||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -105,6 +105,7 @@ pub(super) fn check_session_host(
|
|||||||
"different storage scope".into(),
|
"different storage scope".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
crate::require_subject_in_connection(tx, &host.subject_id)?;
|
||||||
let attr = host
|
let attr = host
|
||||||
.attribution
|
.attribution
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -134,6 +135,7 @@ pub(super) fn check_host(
|
|||||||
"different storage scope".into(),
|
"different storage scope".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
crate::require_subject_in_connection(tx, &host.subject_id)?;
|
||||||
if write {
|
if write {
|
||||||
let state: String = tx.query_row(
|
let state: String = tx.query_row(
|
||||||
"SELECT state FROM subjects WHERE subject_id=?1",
|
"SELECT state FROM subjects WHERE subject_id=?1",
|
||||||
|
|||||||
@@ -349,7 +349,7 @@ mod input_delivery {
|
|||||||
open_action("must not save"),
|
open_action("must not save"),
|
||||||
f.ranges.clone(),
|
f.ranges.clone(),
|
||||||
);
|
);
|
||||||
f.store().retire_subject(&f.host.subject_id).unwrap();
|
crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap();
|
||||||
f.reopen();
|
f.reopen();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges),
|
service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges),
|
||||||
@@ -1210,6 +1210,21 @@ mod snapshot_inputs {
|
|||||||
service::finish(f.store(), &recovery).unwrap(),
|
service::finish(f.store(), &recovery).unwrap(),
|
||||||
saved_completion
|
saved_completion
|
||||||
);
|
);
|
||||||
|
// Deletion, unlike writer release, invalidates even exact saved input/completion replay.
|
||||||
|
f.store()
|
||||||
|
.begin_subject_deletion(&host.subject_id, "delete")
|
||||||
|
.unwrap();
|
||||||
|
subject_deleted(
|
||||||
|
f.store()
|
||||||
|
.prepare_journal_input(&host, &input, generation.job.clone()),
|
||||||
|
);
|
||||||
|
subject_deleted(service::finish(f.store(), &recovery));
|
||||||
|
subject_deleted(JournalJobContext::validated_recovery(
|
||||||
|
f.store(),
|
||||||
|
&host,
|
||||||
|
&generation.id,
|
||||||
|
generation.job.clone(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2833,7 +2848,7 @@ fn job_service_source_reads_are_generation_bound_and_stop_at_supplied_byte_end()
|
|||||||
#[test]
|
#[test]
|
||||||
fn explicit_erasure_remains_available_for_retired_subject_without_reopening_recording() {
|
fn explicit_erasure_remains_available_for_retired_subject_without_reopening_recording() {
|
||||||
let f = Fixture::new();
|
let f = Fixture::new();
|
||||||
f.store().retire_subject(&f.host.subject_id).unwrap();
|
crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap();
|
||||||
assert!(f.store().journal_retention_required(&f.host).unwrap());
|
assert!(f.store().journal_retention_required(&f.host).unwrap());
|
||||||
f.store()
|
f.store()
|
||||||
.erase_journal_session(&f.host, "erase-retired")
|
.erase_journal_session(&f.host, "erase-retired")
|
||||||
@@ -4086,10 +4101,7 @@ fn retired_subject_keeps_reader_access_but_rejects_existing_write_context() {
|
|||||||
},
|
},
|
||||||
vec![],
|
vec![],
|
||||||
);
|
);
|
||||||
fixture
|
crate::tests::legacy_retired_fixture(fixture.store(), &fixture.host.subject_id).unwrap();
|
||||||
.store()
|
|
||||||
.retire_subject(&fixture.host.subject_id)
|
|
||||||
.unwrap();
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
fixture.store().execute_journal(&ctx, late),
|
fixture.store().execute_journal(&ctx, late),
|
||||||
Err(JournalError::PermissionDenied(_))
|
Err(JournalError::PermissionDenied(_))
|
||||||
@@ -5745,3 +5757,311 @@ fn late_gap_admission_and_partial_updates_reject_stale_source_page_cursors() {
|
|||||||
Err(JournalError::InvalidInput(_))
|
Err(JournalError::InvalidInput(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn subject_deleted<T: std::fmt::Debug>(result: JournalResult<T>) {
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
result,
|
||||||
|
Err(JournalError::Subject(
|
||||||
|
crate::SubjektivError::SubjectNotFound(_)
|
||||||
|
))
|
||||||
|
),
|
||||||
|
"unexpected result: {result:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_deletion_fence_denies_all_journal_read_write_binding_and_recovery_paths() {
|
||||||
|
use transport::{
|
||||||
|
JournalJobRequest as Request, JournalSnapshot, JournalTextRead, JournalTextTarget,
|
||||||
|
};
|
||||||
|
let mut f = Fixture::new();
|
||||||
|
let (generation, ctx) = f.generation("delete-job", f.ranges.clone());
|
||||||
|
let operation = f.operation(
|
||||||
|
"saved",
|
||||||
|
open_action("private body"),
|
||||||
|
vec![f.ranges[0].clone()],
|
||||||
|
);
|
||||||
|
let receipt = f.store().execute_journal(&ctx, operation.clone()).unwrap();
|
||||||
|
let entry = receipt.entry_id.unwrap();
|
||||||
|
let reader = JournalHostContext::validated_reader(f.store(), &f.host.subject_id).unwrap();
|
||||||
|
let writer = f
|
||||||
|
.store()
|
||||||
|
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||||
|
.unwrap();
|
||||||
|
let digest = format!("sha256:{}", "a".repeat(64));
|
||||||
|
service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges).unwrap();
|
||||||
|
f.store().database.try_transaction::<_, JournalError>(|tx| {
|
||||||
|
tx.execute("INSERT INTO journal_generation_deliveries(generation_id,source_order,ranges_json) VALUES(?1,0,'[]')", [&generation.id])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
f.store().database.try_transaction::<_, JournalError>(|tx| {
|
||||||
|
tx.execute("INSERT INTO legacy_record_archive(source_table,row_position,row_json) VALUES('subjects',0,?1)", [serde_json::json!({"subject_id":f.host.subject_id,"record_json":"archived private body"}).to_string()])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
f.store()
|
||||||
|
.begin_subject_deletion(&f.host.subject_id, "delete")
|
||||||
|
.unwrap();
|
||||||
|
// Exercise actual SQL owner resolution for *every* indirect content table,
|
||||||
|
// not only table-name inventory. Fence errors must precede PK/immutable/FK errors.
|
||||||
|
f.store()
|
||||||
|
.database
|
||||||
|
.try_with_connection::<_, JournalError>(|tx| {
|
||||||
|
for (table, field, new_owner) in [
|
||||||
|
("journal_sources", "journal_id", "other-journal"),
|
||||||
|
("journal_operations", "journal_id", "other-journal"),
|
||||||
|
("journal_generations", "journal_id", "other-journal"),
|
||||||
|
("journal_session_bindings", "binding_id", "other-binding"),
|
||||||
|
(
|
||||||
|
"journal_generation_deliveries",
|
||||||
|
"generation_id",
|
||||||
|
"other-generation",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"journal_generation_input_deliveries",
|
||||||
|
"generation_id",
|
||||||
|
"other-generation",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"legacy_record_archive",
|
||||||
|
"row_json",
|
||||||
|
"{\"subject_id\":\"other-subject\"}",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r
|
||||||
|
.get::<_, i64>(0))?
|
||||||
|
> 0,
|
||||||
|
"{table}"
|
||||||
|
);
|
||||||
|
let insert = tx
|
||||||
|
.execute(&format!("INSERT INTO {table} SELECT * FROM {table}"), [])
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
insert.to_string().contains("Subject deletion has begun"),
|
||||||
|
"{table}: {insert}"
|
||||||
|
);
|
||||||
|
let update = tx
|
||||||
|
.execute(&format!("UPDATE {table} SET {field}=?1"), [new_owner])
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
update.to_string().contains("Subject deletion has begun"),
|
||||||
|
"{table}: {update}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
macro_rules! denied { ($($result:expr),+ $(,)?) => { $(subject_deleted($result);)+ }; }
|
||||||
|
for host in [&f.host, &reader] {
|
||||||
|
denied!(
|
||||||
|
f.store().list_journals(host, None, 1),
|
||||||
|
f.store().read_journal(host, &f.journal_id),
|
||||||
|
f.store().list_journal_entries(host, &f.journal_id, None, 1),
|
||||||
|
f.store().read_journal_text(
|
||||||
|
host,
|
||||||
|
&f.journal_id,
|
||||||
|
Some(&entry),
|
||||||
|
JournalTextField::Body,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
8
|
||||||
|
),
|
||||||
|
f.store().read_journal_source(host, &f.journal_id, 0),
|
||||||
|
f.store().list_journal_sources(host, &f.journal_id, None, 1),
|
||||||
|
f.store()
|
||||||
|
.query_journal_operation(host, &f.journal_id, "saved"),
|
||||||
|
f.store()
|
||||||
|
.read_journal_operation_status(host, &f.journal_id, "saved"),
|
||||||
|
f.store()
|
||||||
|
.read_journal_operation_coverage(host, &f.journal_id, "saved", 0, 1),
|
||||||
|
f.store()
|
||||||
|
.list_journal_operations(host, &f.journal_id, None, 1),
|
||||||
|
f.store().journal_entry_summary(host, &f.journal_id, &entry),
|
||||||
|
f.store()
|
||||||
|
.journal_correction_targets(host, &f.journal_id, &entry),
|
||||||
|
f.store()
|
||||||
|
.browse_journal_entries(host, None, None, None, None, 1),
|
||||||
|
f.store()
|
||||||
|
.journal_has_unreflected_sources(host, &f.journal_id),
|
||||||
|
f.store().journal_startup_snapshot(host, None),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
denied!(
|
||||||
|
f.store().journal_compaction_snapshot(&f.host),
|
||||||
|
f.store().journal_session_erased(&f.host),
|
||||||
|
f.store().journal_for_session(&f.host),
|
||||||
|
f.store().journal_retention_required(&f.host),
|
||||||
|
f.store().erase_journal_session(&f.host, "erase"),
|
||||||
|
f.store().release_erased_journal_body(&f.host),
|
||||||
|
f.store().journal_predecessor_session(&writer),
|
||||||
|
f.store()
|
||||||
|
.journal_body_is_unpublished(&f.host.subject_id, "runtime", "worker"),
|
||||||
|
f.store()
|
||||||
|
.journal_worker_was_released(&f.host.subject_id, "runtime", "worker"),
|
||||||
|
f.store().journal_writer_active(&writer),
|
||||||
|
f.store()
|
||||||
|
.journal_session_writer_matches(&writer, "source-session"),
|
||||||
|
f.store()
|
||||||
|
.claim_journal_writer(&f.host.subject_id, "runtime", "worker"),
|
||||||
|
f.store()
|
||||||
|
.bind_journal_session(&writer, f.host.attribution.clone().unwrap()),
|
||||||
|
f.store().release_journal_writer(&writer),
|
||||||
|
f.store().accept_journal_sources(&f.host, "late", &f.ranges),
|
||||||
|
f.store()
|
||||||
|
.mark_journal_sources_unavailable(&f.host, &f.journal_id, &f.ranges),
|
||||||
|
f.store()
|
||||||
|
.mark_journal_sources_available(&f.host, &f.journal_id, &f.ranges),
|
||||||
|
f.store().active_journal_generation(&f.host, &f.journal_id),
|
||||||
|
f.store().prepare_journal_generation(
|
||||||
|
&f.host,
|
||||||
|
&f.journal_id,
|
||||||
|
generation.job.clone(),
|
||||||
|
f.ranges.clone()
|
||||||
|
),
|
||||||
|
f.store()
|
||||||
|
.close_journal_generation(&f.host, &generation.id, &generation.job),
|
||||||
|
f.store().read_journal_generation(&f.host, &generation.id),
|
||||||
|
f.store().journal_resume_state(&f.host, &f.journal_id),
|
||||||
|
f.store().execute_journal(&ctx, operation.clone()),
|
||||||
|
JournalJobContext::validated_recovery(
|
||||||
|
f.store(),
|
||||||
|
&f.host,
|
||||||
|
&generation.id,
|
||||||
|
generation.job.clone()
|
||||||
|
),
|
||||||
|
service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges),
|
||||||
|
service::finish(f.store(), &ctx),
|
||||||
|
service::execute_snapshot(f.store(), &ctx, &digest, Request::ReadState),
|
||||||
|
service::execute_snapshot(
|
||||||
|
f.store(),
|
||||||
|
&ctx,
|
||||||
|
&digest,
|
||||||
|
Request::QueryOperation {
|
||||||
|
request_id: "saved".into()
|
||||||
|
}
|
||||||
|
),
|
||||||
|
service::execute_snapshot(
|
||||||
|
f.store(),
|
||||||
|
&ctx,
|
||||||
|
&digest,
|
||||||
|
Request::ReadText {
|
||||||
|
read: JournalTextRead {
|
||||||
|
target: JournalTextTarget::Position {
|
||||||
|
snapshot: JournalSnapshot::Initial
|
||||||
|
},
|
||||||
|
offset: 0,
|
||||||
|
limit: 8,
|
||||||
|
content_token: None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
);
|
||||||
|
// Failed purge and restart must not revive any cached context or exact receipt.
|
||||||
|
f.sql("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected'); END;").unwrap();
|
||||||
|
assert!(
|
||||||
|
f.store()
|
||||||
|
.finish_subject_deletion(&f.host.subject_id, "delete")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
f.reopen();
|
||||||
|
subject_deleted(f.store().execute_journal(&ctx, operation));
|
||||||
|
f.sql("DROP TRIGGER injected_subject_purge_failure;")
|
||||||
|
.unwrap();
|
||||||
|
f.store()
|
||||||
|
.finish_subject_deletion(&f.host.subject_id, "delete")
|
||||||
|
.unwrap();
|
||||||
|
f.store()
|
||||||
|
.database
|
||||||
|
.try_with_connection::<_, JournalError>(|tx| {
|
||||||
|
for table in [
|
||||||
|
"journals",
|
||||||
|
"journal_entries",
|
||||||
|
"journal_corrections",
|
||||||
|
"journal_sources",
|
||||||
|
"journal_generations",
|
||||||
|
"journal_operations",
|
||||||
|
"journal_generation_deliveries",
|
||||||
|
"journal_generation_input_deliveries",
|
||||||
|
"journal_writer_bindings",
|
||||||
|
"journal_session_bindings",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r
|
||||||
|
.get::<_, i64>(0))?,
|
||||||
|
0,
|
||||||
|
"{table}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_deletion_during_capture_rejects_even_empty_delayed_input() {
|
||||||
|
use session_store::capture::CaptureDto;
|
||||||
|
let f = Fixture::new();
|
||||||
|
let result = recording::JournalInput::build(f.store(), &f.host, &f.journal_id, |_, _, _| {
|
||||||
|
f.store()
|
||||||
|
.begin_subject_deletion(&f.host.subject_id, "delete")?;
|
||||||
|
Ok(CaptureDto {
|
||||||
|
version: 1,
|
||||||
|
session_id: "source-session".into(),
|
||||||
|
segment_id: "segment".into(),
|
||||||
|
view_entry_count: 0,
|
||||||
|
entries: vec![],
|
||||||
|
})
|
||||||
|
});
|
||||||
|
subject_deleted(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_deletion_during_source_callback_cannot_return_or_attest_cached_material() {
|
||||||
|
use transport::{JournalJobRequest as Request, JournalTextRead, JournalTextTarget};
|
||||||
|
let f = Fixture::new();
|
||||||
|
let (_, ctx) = f.generation("delayed-source", f.ranges.clone());
|
||||||
|
let result = service::execute_bounded(
|
||||||
|
f.store(),
|
||||||
|
&ctx,
|
||||||
|
Request::ReadText {
|
||||||
|
read: JournalTextRead {
|
||||||
|
target: JournalTextTarget::Source {
|
||||||
|
pointer: f.ranges[0].source.pointer.clone(),
|
||||||
|
},
|
||||||
|
offset: 0,
|
||||||
|
limit: 8,
|
||||||
|
content_token: None,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|source, start, _| {
|
||||||
|
f.store()
|
||||||
|
.begin_subject_deletion(&f.host.subject_id, "delete")?;
|
||||||
|
Ok(recording::JournalMaterial {
|
||||||
|
range: JournalSourceRange {
|
||||||
|
source: source.clone(),
|
||||||
|
bytes: JournalByteRange {
|
||||||
|
start,
|
||||||
|
end: start + 8,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
text: "private!".into(),
|
||||||
|
})
|
||||||
|
},
|
||||||
|
);
|
||||||
|
subject_deleted(result);
|
||||||
|
f.store()
|
||||||
|
.database
|
||||||
|
.try_with_connection::<_, JournalError>(|tx| {
|
||||||
|
assert_eq!(
|
||||||
|
tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM journal_generation_deliveries",
|
||||||
|
[],
|
||||||
|
|r| r.get::<_, i64>(0)
|
||||||
|
)?,
|
||||||
|
0
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|||||||
+63
-28
@@ -1,7 +1,9 @@
|
|||||||
#![doc = include_str!("../README.md")]
|
#![doc = include_str!("../README.md")]
|
||||||
|
|
||||||
pub mod api;
|
pub mod api;
|
||||||
|
pub mod deletion;
|
||||||
pub mod job;
|
pub mod job;
|
||||||
|
pub use deletion::{DeletionSubjectJob, SubjectDeletion};
|
||||||
pub mod journal;
|
pub mod journal;
|
||||||
mod legacy_migrations;
|
mod legacy_migrations;
|
||||||
pub mod permissions;
|
pub mod permissions;
|
||||||
@@ -51,7 +53,9 @@ pub enum SubjektivError {
|
|||||||
InvalidRecord(String),
|
InvalidRecord(String),
|
||||||
#[error("subject `{0}` was not found in this Workspace")]
|
#[error("subject `{0}` was not found in this Workspace")]
|
||||||
SubjectNotFound(String),
|
SubjectNotFound(String),
|
||||||
#[error("subject `{0}` is retired")]
|
#[error("Subject deletion request `{0}` conflicts with a saved deletion")]
|
||||||
|
SubjectDeletionConflict(String),
|
||||||
|
#[error("subject `{0}` is legacy-retired")]
|
||||||
SubjectRetired(String),
|
SubjectRetired(String),
|
||||||
#[error("the selected Worker Profile does not enable subjektiv")]
|
#[error("the selected Worker Profile does not enable subjektiv")]
|
||||||
ProfileDisabled,
|
ProfileDisabled,
|
||||||
@@ -146,7 +150,10 @@ impl<'de> Deserialize<'de> for SubjectRole {
|
|||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum SubjectState {
|
pub enum SubjectState {
|
||||||
Active,
|
Active,
|
||||||
Retired,
|
/// Saved pre-deletion lifecycle state. The old `retired` spelling remains readable
|
||||||
|
/// and serializable solely to preserve existing data; no retirement operation exists.
|
||||||
|
#[serde(rename = "retired")]
|
||||||
|
LegacyRetired,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
@@ -644,6 +651,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
|||||||
"bind Journal generation delivery to immutable Job input digest",
|
"bind Journal generation delivery to immutable Job input digest",
|
||||||
journal::service::migrate_input_delivery,
|
journal::service::migrate_input_delivery,
|
||||||
),
|
),
|
||||||
|
FeatureMigration::new(
|
||||||
|
14,
|
||||||
|
"add permanent Subject deletion fences and explicit erasure",
|
||||||
|
deletion::migrate,
|
||||||
|
),
|
||||||
];
|
];
|
||||||
|
|
||||||
pub const REGISTRATION: FeatureRegistration =
|
pub const REGISTRATION: FeatureRegistration =
|
||||||
@@ -772,6 +784,9 @@ impl SubjektivStore {
|
|||||||
|
|
||||||
pub fn subject(&self, subject_id: &str) -> Result<Option<SubjectRecord>> {
|
pub fn subject(&self, subject_id: &str) -> Result<Option<SubjectRecord>> {
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
if deletion::is_deleting(connection, subject_id)? {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM subjects WHERE subject_id = ?1",
|
"SELECT record_json FROM subjects WHERE subject_id = ?1",
|
||||||
@@ -862,11 +877,13 @@ impl SubjektivStore {
|
|||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
let sql = if after.is_some() {
|
let sql = if after.is_some() {
|
||||||
"SELECT record_json FROM subjects
|
"SELECT record_json FROM subjects
|
||||||
WHERE created_at > ?1 OR (created_at = ?1 AND subject_id > ?2)
|
WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id)
|
||||||
|
AND (created_at > ?1 OR (created_at = ?1 AND subject_id > ?2))
|
||||||
ORDER BY created_at ASC, subject_id ASC
|
ORDER BY created_at ASC, subject_id ASC
|
||||||
LIMIT ?3"
|
LIMIT ?3"
|
||||||
} else {
|
} else {
|
||||||
"SELECT record_json FROM subjects
|
"SELECT record_json FROM subjects
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id)
|
||||||
ORDER BY created_at ASC, subject_id ASC
|
ORDER BY created_at ASC, subject_id ASC
|
||||||
LIMIT ?1"
|
LIMIT ?1"
|
||||||
};
|
};
|
||||||
@@ -895,27 +912,6 @@ impl SubjektivStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Retires a Subject independently of stopping Workers. Memory/history remain unedited, undeleted, and readable.
|
|
||||||
/// Retirement is irreversible; new candidates and Memory changes require an active Subject.
|
|
||||||
pub fn retire_subject(&self, subject_id: &str) -> Result<SubjectRecord> {
|
|
||||||
self.database.try_transaction(|transaction| {
|
|
||||||
let mut subject = require_subject(transaction, subject_id)?;
|
|
||||||
if subject.state == SubjectState::Retired {
|
|
||||||
return Ok(subject);
|
|
||||||
}
|
|
||||||
subject.state = SubjectState::Retired;
|
|
||||||
subject.updated_at = now();
|
|
||||||
let raw = serde_json::to_string(&subject)?;
|
|
||||||
transaction.execute(
|
|
||||||
"UPDATE subjects
|
|
||||||
SET state = 'retired', record_json = ?2, updated_at = ?3
|
|
||||||
WHERE subject_id = ?1",
|
|
||||||
params![subject_id, raw, subject.updated_at],
|
|
||||||
)?;
|
|
||||||
Ok(subject)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Saves an immutable candidate without changing committed Memory.
|
/// Saves an immutable candidate without changing committed Memory.
|
||||||
/// Replay with the same candidate identity and payload returns the original record; different content conflicts.
|
/// Replay with the same candidate identity and payload returns the original record; different content conflicts.
|
||||||
/// Extraction failure after saving never revokes the candidate. The Host commits the extraction pointer separately.
|
/// Extraction failure after saving never revokes the candidate. The Host commits the extraction pointer separately.
|
||||||
@@ -1003,6 +999,10 @@ impl SubjektivStore {
|
|||||||
) -> Result<Option<SubjectSessionAttribution>> {
|
) -> Result<Option<SubjectSessionAttribution>> {
|
||||||
validate_label("session id", session_id)?;
|
validate_label("session id", session_id)?;
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
if let Some(subject) = deletion::deleted_session_subject(connection, session_id)? {
|
||||||
|
return Err(SubjektivError::SubjectNotFound(subject));
|
||||||
|
}
|
||||||
|
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM subject_session_attributions
|
"SELECT record_json FROM subject_session_attributions
|
||||||
@@ -1011,7 +1011,12 @@ impl SubjektivStore {
|
|||||||
|row| row.get::<_, String>(0),
|
|row| row.get::<_, String>(0),
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
raw.map(|raw| parse_session_attribution(&raw)).transpose()
|
raw.map(|raw| {
|
||||||
|
let record = parse_session_attribution(&raw)?;
|
||||||
|
deletion::require_not_deleting(connection, &record.subject_id)?;
|
||||||
|
Ok(record)
|
||||||
|
})
|
||||||
|
.transpose()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1021,6 +1026,7 @@ impl SubjektivStore {
|
|||||||
) -> Result<Vec<SubjectSessionAttribution>> {
|
) -> Result<Vec<SubjectSessionAttribution>> {
|
||||||
validate_label("subject id", subject_id)?;
|
validate_label("subject id", subject_id)?;
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
require_subject_in_connection(connection, subject_id)?;
|
||||||
let mut statement = connection.prepare(
|
let mut statement = connection.prepare(
|
||||||
"SELECT record_json FROM subject_session_attributions
|
"SELECT record_json FROM subject_session_attributions
|
||||||
WHERE subject_id = ?1
|
WHERE subject_id = ?1
|
||||||
@@ -1062,6 +1068,7 @@ impl SubjektivStore {
|
|||||||
}
|
}
|
||||||
let query_limit = limit.saturating_add(1);
|
let query_limit = limit.saturating_add(1);
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
require_subject_in_connection(connection, subject_id)?;
|
||||||
let raw = match (session_id, after) {
|
let raw = match (session_id, after) {
|
||||||
(Some(session_id), _) => {
|
(Some(session_id), _) => {
|
||||||
let mut statement = connection.prepare(
|
let mut statement = connection.prepare(
|
||||||
@@ -1128,6 +1135,7 @@ impl SubjektivStore {
|
|||||||
candidate_id: &str,
|
candidate_id: &str,
|
||||||
) -> Result<Option<SubjectStagingRecord>> {
|
) -> Result<Option<SubjectStagingRecord>> {
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
deletion::require_not_deleting(connection, subject_id)?;
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM staging_records
|
"SELECT record_json FROM staging_records
|
||||||
@@ -1292,6 +1300,7 @@ impl SubjektivStore {
|
|||||||
validate_nonempty("resolution reason", &request.reason)?;
|
validate_nonempty("resolution reason", &request.reason)?;
|
||||||
let request_raw = serde_json::to_string(&request)?;
|
let request_raw = serde_json::to_string(&request)?;
|
||||||
self.database.try_transaction(|transaction| {
|
self.database.try_transaction(|transaction| {
|
||||||
|
deletion::require_not_deleting(transaction, subject_id)?;
|
||||||
if let Some((stored_request, stored_receipt)) = transaction
|
if let Some((stored_request, stored_receipt)) = transaction
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT request_json, receipt_json
|
"SELECT request_json, receipt_json
|
||||||
@@ -1426,6 +1435,7 @@ impl SubjektivStore {
|
|||||||
candidate_id: &str,
|
candidate_id: &str,
|
||||||
) -> Result<Option<StagingResolution>> {
|
) -> Result<Option<StagingResolution>> {
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
deletion::require_not_deleting(connection, subject_id)?;
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT resolution_json FROM staging_resolutions
|
"SELECT resolution_json FROM staging_resolutions
|
||||||
@@ -1440,6 +1450,7 @@ impl SubjektivStore {
|
|||||||
|
|
||||||
pub fn memory(&self, subject_id: &str, memory_id: &str) -> Result<Option<MemoryRecord>> {
|
pub fn memory(&self, subject_id: &str, memory_id: &str) -> Result<Option<MemoryRecord>> {
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
deletion::require_not_deleting(connection, subject_id)?;
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM memory_records
|
"SELECT record_json FROM memory_records
|
||||||
@@ -1459,6 +1470,7 @@ impl SubjektivStore {
|
|||||||
change_id: String,
|
change_id: String,
|
||||||
) -> Result<Option<MemoryRecord>> {
|
) -> Result<Option<MemoryRecord>> {
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
deletion::require_not_deleting(connection, subject_id)?;
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM memory_changes
|
"SELECT record_json FROM memory_changes
|
||||||
@@ -1854,6 +1866,7 @@ impl SubjektivStore {
|
|||||||
snapshot_id: &str,
|
snapshot_id: &str,
|
||||||
) -> Result<Option<SurfaceSnapshot>> {
|
) -> Result<Option<SurfaceSnapshot>> {
|
||||||
self.database.try_with_connection(|connection| {
|
self.database.try_with_connection(|connection| {
|
||||||
|
deletion::require_not_deleting(connection, subject_id)?;
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT snapshot_json FROM surface_snapshots
|
"SELECT snapshot_json FROM surface_snapshots
|
||||||
@@ -2427,6 +2440,11 @@ fn write_session_attribution(
|
|||||||
record: SubjectSessionAttribution,
|
record: SubjectSessionAttribution,
|
||||||
raw: String,
|
raw: String,
|
||||||
) -> Result<SubjectSessionAttribution> {
|
) -> Result<SubjectSessionAttribution> {
|
||||||
|
deletion::require_live_worker(transaction, &record.runtime_id, &record.worker_id)?;
|
||||||
|
if let Some(subject) = deletion::deleted_session_subject(transaction, &record.session_id)? {
|
||||||
|
return Err(SubjektivError::SubjectNotFound(subject));
|
||||||
|
}
|
||||||
|
|
||||||
let existing = transaction
|
let existing = transaction
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM subject_session_attributions WHERE session_id = ?1",
|
"SELECT record_json FROM subject_session_attributions WHERE session_id = ?1",
|
||||||
@@ -2621,6 +2639,7 @@ fn require_subject_in_connection(
|
|||||||
connection: &rusqlite::Connection,
|
connection: &rusqlite::Connection,
|
||||||
subject_id: &str,
|
subject_id: &str,
|
||||||
) -> Result<SubjectRecord> {
|
) -> Result<SubjectRecord> {
|
||||||
|
deletion::require_not_deleting(connection, subject_id)?;
|
||||||
let raw = connection
|
let raw = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM subjects WHERE subject_id = ?1",
|
"SELECT record_json FROM subjects WHERE subject_id = ?1",
|
||||||
@@ -2637,7 +2656,7 @@ fn require_active_subject(
|
|||||||
subject_id: &str,
|
subject_id: &str,
|
||||||
) -> Result<SubjectRecord> {
|
) -> Result<SubjectRecord> {
|
||||||
let subject = require_subject(transaction, subject_id)?;
|
let subject = require_subject(transaction, subject_id)?;
|
||||||
if subject.state == SubjectState::Retired {
|
if subject.state == SubjectState::LegacyRetired {
|
||||||
return Err(SubjektivError::SubjectRetired(subject_id.to_string()));
|
return Err(SubjektivError::SubjectRetired(subject_id.to_string()));
|
||||||
}
|
}
|
||||||
Ok(subject)
|
Ok(subject)
|
||||||
@@ -3278,6 +3297,7 @@ fn require_job_generation(
|
|||||||
job_id: &str,
|
job_id: &str,
|
||||||
attempt_id: &str,
|
attempt_id: &str,
|
||||||
) -> Result<String> {
|
) -> Result<String> {
|
||||||
|
deletion::require_not_deleting(connection, subject_id)?;
|
||||||
let change_id: Option<String> = connection.query_row(
|
let change_id: Option<String> = connection.query_row(
|
||||||
"SELECT memory_fingerprint FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4",
|
"SELECT memory_fingerprint FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4",
|
||||||
params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0),
|
params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0),
|
||||||
@@ -3370,6 +3390,21 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use feature_storage::FeatureStorage;
|
use feature_storage::FeatureStorage;
|
||||||
|
|
||||||
|
/// Builds saved pre-deletion lifecycle data; not a current product operation.
|
||||||
|
pub(crate) fn legacy_retired_fixture(
|
||||||
|
store: &SubjektivStore,
|
||||||
|
subject_id: &str,
|
||||||
|
) -> Result<SubjectRecord> {
|
||||||
|
store.database.try_transaction(|tx| {
|
||||||
|
let mut subject = require_subject(tx, subject_id)?;
|
||||||
|
subject.state = SubjectState::LegacyRetired;
|
||||||
|
subject.updated_at = now();
|
||||||
|
tx.execute("UPDATE subjects SET state='retired',record_json=?2,updated_at=?3 WHERE subject_id=?1",
|
||||||
|
params![subject_id,serde_json::to_string(&subject)?,subject.updated_at])?;
|
||||||
|
Ok(subject)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn role() -> SubjectRole {
|
pub(super) fn role() -> SubjectRole {
|
||||||
SubjectRole::new("workspace_companion").unwrap()
|
SubjectRole::new("workspace_companion").unwrap()
|
||||||
}
|
}
|
||||||
@@ -5047,8 +5082,8 @@ mod tests {
|
|||||||
Err(SubjektivError::InvalidStateTransition { .. })
|
Err(SubjektivError::InvalidStateTransition { .. })
|
||||||
));
|
));
|
||||||
|
|
||||||
let retired = store.retire_subject(&subject.id).unwrap();
|
let retired = crate::tests::legacy_retired_fixture(&store, &subject.id).unwrap();
|
||||||
assert_eq!(retired.state, SubjectState::Retired);
|
assert_eq!(retired.state, SubjectState::LegacyRetired);
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
store.create_memory(&subject.id, draft("after retirement", "invalid")),
|
store.create_memory(&subject.id, draft("after retirement", "invalid")),
|
||||||
Err(SubjektivError::SubjectRetired(_))
|
Err(SubjektivError::SubjectRetired(_))
|
||||||
|
|||||||
@@ -94,6 +94,10 @@ impl SubjektivStore {
|
|||||||
pub fn bind_worker_execution_grant(&self, grant: &WorkerExecutionGrant) -> Result<()> {
|
pub fn bind_worker_execution_grant(&self, grant: &WorkerExecutionGrant) -> Result<()> {
|
||||||
validate_grant(self.scope_id(), grant)?;
|
validate_grant(self.scope_id(), grant)?;
|
||||||
self.database.try_transaction(|tx| {
|
self.database.try_transaction(|tx| {
|
||||||
|
require_subject(tx, &grant.subject_id)?;
|
||||||
|
for (runtime, worker) in [(&grant.runtime_id, &grant.worker_id), (&grant.origin_runtime_id, &grant.origin_worker_id)] {
|
||||||
|
if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)", params![runtime,worker], |r|r.get::<_,bool>(0))? { return Err(denied("Worker belongs to a deleted Subject")); }
|
||||||
|
}
|
||||||
if let Some(existing) = grant_in_connection(tx, &grant.runtime_id, &grant.worker_id)? {
|
if let Some(existing) = grant_in_connection(tx, &grant.runtime_id, &grant.worker_id)? {
|
||||||
return if existing == *grant { Ok(()) } else {
|
return if existing == *grant { Ok(()) } else {
|
||||||
Err(SubjektivError::WorkerExecutionGrantConflict(format!("{}/{}", grant.runtime_id, grant.worker_id)))
|
Err(SubjektivError::WorkerExecutionGrantConflict(format!("{}/{}", grant.runtime_id, grant.worker_id)))
|
||||||
@@ -184,9 +188,12 @@ fn grant_in_connection(
|
|||||||
runtime: &str,
|
runtime: &str,
|
||||||
worker: &str,
|
worker: &str,
|
||||||
) -> Result<Option<WorkerExecutionGrant>> {
|
) -> Result<Option<WorkerExecutionGrant>> {
|
||||||
|
// Absence must never become non-Subject fallback authority after purge.
|
||||||
|
deletion::require_live_worker(connection, runtime, worker)?;
|
||||||
|
|
||||||
let raw: Option<String> = connection
|
let raw: Option<String> = connection
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2",
|
"SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2 AND NOT EXISTS(SELECT 1 FROM subject_deletions d WHERE d.subject_id=worker_execution_grants.subject_id)",
|
||||||
params![runtime, worker],
|
params![runtime, worker],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -315,7 +315,7 @@ fn redelegation_requires_current_source_and_explicit_retained_parent_ceiling() {
|
|||||||
store.worker_execution_grant("runtime", "child").unwrap(),
|
store.worker_execution_grant("runtime", "child").unwrap(),
|
||||||
Some(child)
|
Some(child)
|
||||||
);
|
);
|
||||||
store.retire_subject(&subject).unwrap();
|
crate::tests::legacy_retired_fixture(&store, &subject).unwrap();
|
||||||
stale.source_change_id = revoked.change_id;
|
stale.source_change_id = revoked.change_id;
|
||||||
assert!(store.bind_worker_execution_grant(&stale).is_err());
|
assert!(store.bind_worker_execution_grant(&stale).is_err());
|
||||||
}
|
}
|
||||||
@@ -582,7 +582,7 @@ fn subject_grant_listing_is_scoped_retained_ordered_and_fails_on_overflow() {
|
|||||||
&update("revoke", Some(&source.change_id), Vec::new()),
|
&update("revoke", Some(&source.change_id), Vec::new()),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
store.retire_subject(&subject).unwrap();
|
crate::tests::legacy_retired_fixture(&store, &subject).unwrap();
|
||||||
let listed = store
|
let listed = store
|
||||||
.list_subject_worker_execution_grants(&subject)
|
.list_subject_worker_execution_grants(&subject)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -654,3 +654,51 @@ fn failed_schema_migration_rolls_back_new_tables_and_preserves_old_data() {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deletion_fences_source_and_child_grant_replays_and_never_falls_back_after_purge() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
|
||||||
|
let subject = store.create_subject(role()).unwrap().id;
|
||||||
|
let replacement = update("allow", None, vec![permission("file.read", ".", true)]);
|
||||||
|
let source = store
|
||||||
|
.set_subject_permissions(&subject, &replacement)
|
||||||
|
.unwrap();
|
||||||
|
let child = grant(&subject, &source.change_id, "child");
|
||||||
|
store.bind_worker_execution_grant(&child).unwrap();
|
||||||
|
store.begin_subject_deletion(&subject, "delete").unwrap();
|
||||||
|
assert!(store.subject_permissions(&subject).is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.set_subject_permissions(&subject, &replacement)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(store.bind_worker_execution_grant(&child).is_err());
|
||||||
|
assert!(store.worker_execution_grant("runtime", "child").is_err());
|
||||||
|
assert!(store.worker_subject_deleted("runtime", "body").unwrap());
|
||||||
|
assert!(store.worker_subject_deleted("runtime", "child").unwrap());
|
||||||
|
assert_eq!(store.deletion_subject_workers(&subject).unwrap().len(), 2);
|
||||||
|
store.finish_subject_deletion(&subject, "delete").unwrap();
|
||||||
|
assert!(store.worker_execution_grant("runtime", "child").is_err());
|
||||||
|
assert!(store.worker_execution_grant("runtime", "body").is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_execution_grant("runtime", "unrelated")
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let fresh = store.create_subject(role()).unwrap().id;
|
||||||
|
assert!(store.subject_permissions(&fresh).unwrap().is_none());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.record_session_attribution(
|
||||||
|
SubjectSessionAttribution::new(&fresh, "runtime", "child", "new-session").unwrap()
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.claim_journal_writer(&fresh, "runtime", "body")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -727,6 +727,23 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
Err("execution backend cannot confirm Subject create execution absence".into())
|
Err("execution backend cannot confirm Subject create execution absence".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read-only exact execution identity check before Runtime persists exclusion.
|
||||||
|
fn preflight_subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Err("execution backend cannot validate Subject Worker identity".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runtime has durably fenced admission. Confirm shutdown and discard all
|
||||||
|
/// execution candidates under the backend Worker lock; uncertainty must fail.
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Err("execution backend cannot confirm Subject Worker shutdown".into())
|
||||||
|
}
|
||||||
|
|
||||||
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
|
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
|
||||||
fn subject_release(
|
fn subject_release(
|
||||||
&self,
|
&self,
|
||||||
@@ -1069,6 +1086,20 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.recover_subject_create(request)
|
self.backend.recover_subject_create(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn preflight_subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.backend.preflight_subject_delete_worker(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.backend.subject_delete_worker(request)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn subject_release(
|
pub(crate) fn subject_release(
|
||||||
&self,
|
&self,
|
||||||
request: &runtime_api::SubjectReleaseRequest,
|
request: &runtime_api::SubjectReleaseRequest,
|
||||||
|
|||||||
@@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
|
|||||||
#[path = "test_support.rs"]
|
#[path = "test_support.rs"]
|
||||||
pub mod test_support;
|
pub mod test_support;
|
||||||
|
|
||||||
pub(crate) const SCHEMA_VERSION: u32 = 14;
|
pub(crate) const SCHEMA_VERSION: u32 = 15;
|
||||||
|
const RESTORE_REPAIR_SCHEMA_VERSION: u32 = 14;
|
||||||
const RESTORE_DECISION_SCHEMA_VERSION: u32 = 13;
|
const RESTORE_DECISION_SCHEMA_VERSION: u32 = 13;
|
||||||
const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12;
|
const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12;
|
||||||
// Frozen source schemas: do not move their meaning when adding a migration.
|
// Frozen source schemas: do not move their meaning when adding a migration.
|
||||||
@@ -560,6 +561,22 @@ impl FsRuntimeStore {
|
|||||||
self.root.join(RUNTIME_FILE)
|
self.root.join(RUNTIME_FILE)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn worker_originals_absent(
|
||||||
|
&self,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
) -> Result<bool, RuntimeError> {
|
||||||
|
let path = self.worker_dir(&worker_id);
|
||||||
|
match fs::symlink_metadata(&path) {
|
||||||
|
Ok(_) => Ok(false),
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true),
|
||||||
|
Err(source) => Err(RuntimeError::StoreIo {
|
||||||
|
operation: "confirm Subject Worker original absence",
|
||||||
|
path,
|
||||||
|
source,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf {
|
fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf {
|
||||||
self.root.join(WORKERS_DIR).join(worker_id.to_string())
|
self.root.join(WORKERS_DIR).join(worker_id.to_string())
|
||||||
}
|
}
|
||||||
@@ -659,6 +676,8 @@ pub(crate) struct OpenedFsRuntimeStore {
|
|||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub(crate) struct PersistedRuntimeState {
|
pub(crate) struct PersistedRuntimeState {
|
||||||
|
pub(crate) subject_delete_worker_fences:
|
||||||
|
BTreeMap<WorkerId, crate::runtime::SubjectDeleteWorkerFence>,
|
||||||
pub(crate) subject_create_recovery_fences:
|
pub(crate) subject_create_recovery_fences:
|
||||||
BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
|
BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
|
||||||
pub(crate) subject_releases: Vec<(
|
pub(crate) subject_releases: Vec<(
|
||||||
@@ -876,11 +895,12 @@ fn plan_runtime_store_migration(
|
|||||||
| SOURCE_SCHEMA_VERSION
|
| SOURCE_SCHEMA_VERSION
|
||||||
| RESTORE_ADMISSION_SCHEMA_VERSION
|
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||||
| RESTORE_DECISION_SCHEMA_VERSION
|
| RESTORE_DECISION_SCHEMA_VERSION
|
||||||
|
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||||
) {
|
) {
|
||||||
return Err(runtime_store_corrupt(
|
return Err(runtime_store_corrupt(
|
||||||
&runtime_path,
|
&runtime_path,
|
||||||
format!(
|
format!(
|
||||||
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
|
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, {RESTORE_REPAIR_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
|
||||||
),
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@@ -1216,6 +1236,7 @@ fn migrate_worker_record(
|
|||||||
| SOURCE_SCHEMA_VERSION
|
| SOURCE_SCHEMA_VERSION
|
||||||
| RESTORE_ADMISSION_SCHEMA_VERSION
|
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||||
| RESTORE_DECISION_SCHEMA_VERSION
|
| RESTORE_DECISION_SCHEMA_VERSION
|
||||||
|
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||||
) {
|
) {
|
||||||
// Admission identity survives migration. Older pending records did not
|
// Admission identity survives migration. Older pending records did not
|
||||||
// distinguish a committed decision from preparation or failed cleanup.
|
// distinguish a committed decision from preparation or failed cleanup.
|
||||||
@@ -1229,10 +1250,12 @@ fn migrate_worker_record(
|
|||||||
"Worker schema does not match Runtime migration source".into(),
|
"Worker schema does not match Runtime migration source".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if source_schema_version != RESTORE_DECISION_SCHEMA_VERSION {
|
if source_schema_version < RESTORE_DECISION_SCHEMA_VERSION {
|
||||||
migrate_restore_commit_fact(&mut identity_document, identity_path)?;
|
migrate_restore_commit_fact(&mut identity_document, identity_path)?;
|
||||||
}
|
}
|
||||||
migrate_restore_repair_fact(&mut identity_document, identity_path)?;
|
if source_schema_version < RESTORE_REPAIR_SCHEMA_VERSION {
|
||||||
|
migrate_restore_repair_fact(&mut identity_document, identity_path)?;
|
||||||
|
}
|
||||||
identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION);
|
identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION);
|
||||||
let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document)
|
let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document)
|
||||||
.map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?;
|
.map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?;
|
||||||
@@ -1691,6 +1714,9 @@ fn migrate_runtime_document(
|
|||||||
"schema_version".to_string(),
|
"schema_version".to_string(),
|
||||||
serde_json::Value::from(SCHEMA_VERSION),
|
serde_json::Value::from(SCHEMA_VERSION),
|
||||||
);
|
);
|
||||||
|
object
|
||||||
|
.entry("subject_delete_worker_fences")
|
||||||
|
.or_insert_with(|| serde_json::json!({}));
|
||||||
// Intermediate schema-11 writers may already have saved irreversible fences.
|
// Intermediate schema-11 writers may already have saved irreversible fences.
|
||||||
// Default only a missing field; malformed or existing fences must not be erased.
|
// Default only a missing field; malformed or existing fences must not be erased.
|
||||||
object
|
object
|
||||||
@@ -1698,7 +1724,10 @@ fn migrate_runtime_document(
|
|||||||
.or_insert_with(|| serde_json::json!({}));
|
.or_insert_with(|| serde_json::json!({}));
|
||||||
if !matches!(
|
if !matches!(
|
||||||
source_schema_version,
|
source_schema_version,
|
||||||
SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION
|
SOURCE_SCHEMA_VERSION
|
||||||
|
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||||
|
| RESTORE_DECISION_SCHEMA_VERSION
|
||||||
|
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||||
) {
|
) {
|
||||||
object.remove("workers");
|
object.remove("workers");
|
||||||
// This legacy cache is not Runtime lifecycle or restore authority. Bundles
|
// This legacy cache is not Runtime lifecycle or restore authority. Bundles
|
||||||
@@ -2249,6 +2278,7 @@ fn migrate_runtime_store_in_place(
|
|||||||
SOURCE_SCHEMA_VERSION
|
SOURCE_SCHEMA_VERSION
|
||||||
| RESTORE_ADMISSION_SCHEMA_VERSION
|
| RESTORE_ADMISSION_SCHEMA_VERSION
|
||||||
| RESTORE_DECISION_SCHEMA_VERSION
|
| RESTORE_DECISION_SCHEMA_VERSION
|
||||||
|
| RESTORE_REPAIR_SCHEMA_VERSION
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
atomic_write_json(
|
atomic_write_json(
|
||||||
@@ -2288,6 +2318,7 @@ fn migrate_runtime_store_in_place(
|
|||||||
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
struct RuntimeSnapshot {
|
struct RuntimeSnapshot {
|
||||||
|
subject_delete_worker_fences: BTreeMap<WorkerId, crate::runtime::SubjectDeleteWorkerFence>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
subject_create_recovery_fences: BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
|
subject_create_recovery_fences: BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -2326,6 +2357,7 @@ fn record_worker_load_diagnostic(
|
|||||||
impl RuntimeSnapshot {
|
impl RuntimeSnapshot {
|
||||||
fn from_persisted(state: &PersistedRuntimeState) -> Self {
|
fn from_persisted(state: &PersistedRuntimeState) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
subject_delete_worker_fences: state.subject_delete_worker_fences.clone(),
|
||||||
subject_create_recovery_fences: state.subject_create_recovery_fences.clone(),
|
subject_create_recovery_fences: state.subject_create_recovery_fences.clone(),
|
||||||
subject_releases: state.subject_releases.clone(),
|
subject_releases: state.subject_releases.clone(),
|
||||||
schema_version: SCHEMA_VERSION,
|
schema_version: SCHEMA_VERSION,
|
||||||
@@ -2350,6 +2382,33 @@ impl RuntimeSnapshot {
|
|||||||
),
|
),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
for (worker_id, fence) in &self.subject_delete_worker_fences {
|
||||||
|
let request = &fence.request;
|
||||||
|
let response_matches = fence.response.as_ref().is_none_or(|response| {
|
||||||
|
response.worker_id == request.worker_id
|
||||||
|
&& response.workspace_id == request.workspace_id
|
||||||
|
&& response.create_fingerprint == request.create_fingerprint
|
||||||
|
&& response.operation_id == request.operation_id
|
||||||
|
});
|
||||||
|
if *worker_id != request.worker_id
|
||||||
|
|| request.workspace_id.trim().is_empty()
|
||||||
|
|| request.create_fingerprint.trim().is_empty()
|
||||||
|
|| request.create_fingerprint.len() > 512
|
||||||
|
|| request.operation_id.trim().is_empty()
|
||||||
|
|| request.operation_id.len() > 512
|
||||||
|
|| self
|
||||||
|
.workspace_owners
|
||||||
|
.get(&request.workspace_id)
|
||||||
|
.is_none_or(|owner| owner.trim().is_empty())
|
||||||
|
|| !response_matches
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::StoreCorrupt {
|
||||||
|
operation: "read Subject delete Worker fence",
|
||||||
|
path: path.to_path_buf(),
|
||||||
|
message: "invalid deletion identity, receipt or Workspace owner".into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
for (worker_id, fence) in &self.subject_create_recovery_fences {
|
for (worker_id, fence) in &self.subject_create_recovery_fences {
|
||||||
let request = &fence.request;
|
let request = &fence.request;
|
||||||
let response_matches = fence.response.as_ref().is_none_or(|response| {
|
let response_matches = fence.response.as_ref().is_none_or(|response| {
|
||||||
@@ -2392,6 +2451,7 @@ impl RuntimeSnapshot {
|
|||||||
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
|
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
|
||||||
) -> PersistedRuntimeState {
|
) -> PersistedRuntimeState {
|
||||||
PersistedRuntimeState {
|
PersistedRuntimeState {
|
||||||
|
subject_delete_worker_fences: self.subject_delete_worker_fences,
|
||||||
subject_create_recovery_fences: self.subject_create_recovery_fences,
|
subject_create_recovery_fences: self.subject_create_recovery_fences,
|
||||||
subject_releases: self.subject_releases,
|
subject_releases: self.subject_releases,
|
||||||
display_name: self.display_name,
|
display_name: self.display_name,
|
||||||
@@ -3218,8 +3278,7 @@ mod tests {
|
|||||||
assert_eq!(fs::read(&worker_path).unwrap(), worker_before);
|
assert_eq!(fs::read(&worker_path).unwrap(), worker_before);
|
||||||
// The direct reader still rejects a previous schema: migration is
|
// The direct reader still rejects a previous schema: migration is
|
||||||
// explicit, not permissive deserialization of a newer format.
|
// explicit, not permissive deserialization of a newer format.
|
||||||
let snapshot: RuntimeSnapshot = serde_json::from_value(runtime.clone()).unwrap();
|
assert!(serde_json::from_value::<RuntimeSnapshot>(runtime.clone()).is_err());
|
||||||
assert!(snapshot.validate(&runtime_path).is_err());
|
|
||||||
|
|
||||||
let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap();
|
let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap();
|
||||||
let state = opened.state.as_ref().unwrap();
|
let state = opened.state.as_ref().unwrap();
|
||||||
@@ -3231,6 +3290,7 @@ mod tests {
|
|||||||
expected_worker
|
expected_worker
|
||||||
);
|
);
|
||||||
runtime["schema_version"] = serde_json::json!(SCHEMA_VERSION);
|
runtime["schema_version"] = serde_json::json!(SCHEMA_VERSION);
|
||||||
|
runtime["subject_delete_worker_fences"] = serde_json::json!({});
|
||||||
if fence_state == "absent" {
|
if fence_state == "absent" {
|
||||||
runtime["subject_create_recovery_fences"] = serde_json::json!({});
|
runtime["subject_create_recovery_fences"] = serde_json::json!({});
|
||||||
}
|
}
|
||||||
@@ -4072,6 +4132,8 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
include!("subject_delete_store_tests.rs");
|
||||||
|
|
||||||
fn canonical_worker_store() -> (
|
fn canonical_worker_store() -> (
|
||||||
tempfile::TempDir,
|
tempfile::TempDir,
|
||||||
FsRuntimeStore,
|
FsRuntimeStore,
|
||||||
|
|||||||
@@ -2102,7 +2102,11 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
|||||||
{
|
{
|
||||||
return Some("workers:input");
|
return Some("workers:input");
|
||||||
}
|
}
|
||||||
if path == "/v1/subject-create-recovery" && *method == Method::POST {
|
if matches!(
|
||||||
|
path,
|
||||||
|
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
|
||||||
|
) && *method == Method::POST
|
||||||
|
{
|
||||||
return Some("workers:delete");
|
return Some("workers:delete");
|
||||||
}
|
}
|
||||||
if path.ends_with("/stop")
|
if path.ends_with("/stop")
|
||||||
@@ -3332,6 +3336,18 @@ mod tests {
|
|||||||
struct AcceptingBackend;
|
struct AcceptingBackend;
|
||||||
|
|
||||||
impl WorkerExecutionBackend for AcceptingBackend {
|
impl WorkerExecutionBackend for AcceptingBackend {
|
||||||
|
fn preflight_subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
fn recover_subject_create(
|
fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
_request: &runtime_api::SubjectCreateRecoveryRequest,
|
_request: &runtime_api::SubjectCreateRecoveryRequest,
|
||||||
|
|||||||
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<runtime_api::SubjectDeleteWorkerResponse, runtime_api::RuntimeApiError> {
|
||||||
|
let scope = material_scope(&self.state, &request.workspace_id)?;
|
||||||
|
self.state
|
||||||
|
.runtime
|
||||||
|
.subject_delete_worker_scoped(&scope, request)
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)
|
||||||
|
}
|
||||||
|
|
||||||
async fn recover_subject_create(
|
async fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||||
|
|||||||
@@ -1222,6 +1222,54 @@ async fn subject_create_recovery_signed_http_requires_delete_and_matching_worksp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn subject_delete_worker_signed_http_requires_delete_and_matching_workspace() {
|
||||||
|
let f = Fixture::new();
|
||||||
|
let identity = RuntimeIdentityMaterial::generate("recovery-key").unwrap();
|
||||||
|
let trust = workspace_trust(&identity);
|
||||||
|
let app = runtime_http_router_with_workspace_auth(
|
||||||
|
f.runtime.clone(),
|
||||||
|
None,
|
||||||
|
workspace_test_auth(vec![trust.clone()]),
|
||||||
|
);
|
||||||
|
let path = "/v1/subject-delete-worker";
|
||||||
|
let mut request = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: f.scope.workspace_id.clone(),
|
||||||
|
worker_id: WorkerId::now_v7(),
|
||||||
|
create_fingerprint: "unpublished-create".into(),
|
||||||
|
operation_id: "recovery-1".into(),
|
||||||
|
};
|
||||||
|
for operation in ["workers:read", "workers:stop", "workers:create"] {
|
||||||
|
let body = serde_json::to_vec(&request).unwrap();
|
||||||
|
let mut claims = signed_claims(&trust, "POST", path, &body);
|
||||||
|
claims.operation = operation.into();
|
||||||
|
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
}
|
||||||
|
request.workspace_id = "other-workspace".into();
|
||||||
|
let body = serde_json::to_vec(&request).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", path, &body);
|
||||||
|
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||||
|
request.workspace_id = f.scope.workspace_id.clone();
|
||||||
|
for _ in 0..2 {
|
||||||
|
let body = serde_json::to_vec(&request).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", path, &body);
|
||||||
|
assert_eq!(claims.operation, "workers:delete");
|
||||||
|
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let bytes = axum::body::to_bytes(response.into_body(), 16384)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let receipt: runtime_api::SubjectDeleteWorkerResponse =
|
||||||
|
serde_json::from_slice(&bytes).unwrap();
|
||||||
|
assert_eq!(receipt.worker_id, request.worker_id);
|
||||||
|
assert_eq!(receipt.workspace_id, request.workspace_id);
|
||||||
|
assert_eq!(receipt.create_fingerprint, request.create_fingerprint);
|
||||||
|
assert_eq!(receipt.operation_id, request.operation_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() {
|
async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() {
|
||||||
let f = Fixture::new();
|
let f = Fixture::new();
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ fn runtime_fixture(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId)
|
|||||||
"schema_version": schema, "display_name": "retained Runtime",
|
"schema_version": schema, "display_name": "retained Runtime",
|
||||||
"backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
|
"backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
|
||||||
"config_bundles": {}, "workspace_owners": {"workspace-a": "owner-a"},
|
"config_bundles": {}, "workspace_owners": {"workspace-a": "owner-a"},
|
||||||
"diagnostics": [], "subject_releases": [],
|
"diagnostics": [], "subject_releases": [], "subject_delete_worker_fences": {},
|
||||||
"subject_create_recovery_fences": {fenced_worker.to_string(): {
|
"subject_create_recovery_fences": {fenced_worker.to_string(): {
|
||||||
"request": {"workspace_id": "workspace-a", "worker_id": fenced_worker,
|
"request": {"workspace_id": "workspace-a", "worker_id": fenced_worker,
|
||||||
"create_fingerprint": "retained-create", "operation_id": "retained-recovery"},
|
"create_fingerprint": "retained-create", "operation_id": "retained-recovery"},
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
mod subject_delete;
|
||||||
|
pub(crate) use subject_delete::SubjectDeleteWorkerFence;
|
||||||
#[cfg(feature = "restore-checkpoints")]
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
mod restore_checkpoint;
|
mod restore_checkpoint;
|
||||||
mod restore_continuation;
|
mod restore_continuation;
|
||||||
@@ -5645,6 +5647,7 @@ struct RuntimeState {
|
|||||||
// Ephemeral write barrier, not authority: only canonical reread + sync can
|
// Ephemeral write barrier, not authority: only canonical reread + sync can
|
||||||
// release it. Prevents another mutation from persisting an older projection.
|
// release it. Prevents another mutation from persisting an older projection.
|
||||||
unreconciled_worker_writes: std::collections::BTreeSet<WorkerId>,
|
unreconciled_worker_writes: std::collections::BTreeSet<WorkerId>,
|
||||||
|
subject_delete_worker_fences: BTreeMap<WorkerId, SubjectDeleteWorkerFence>,
|
||||||
subject_create_recovery_fences: BTreeMap<WorkerId, SubjectCreateRecoveryFence>,
|
subject_create_recovery_fences: BTreeMap<WorkerId, SubjectCreateRecoveryFence>,
|
||||||
subject_release_operations: BTreeMap<
|
subject_release_operations: BTreeMap<
|
||||||
(WorkerId, String),
|
(WorkerId, String),
|
||||||
@@ -5686,6 +5689,7 @@ impl RuntimeState {
|
|||||||
next_diagnostic_id: 1,
|
next_diagnostic_id: 1,
|
||||||
workers: BTreeMap::new(),
|
workers: BTreeMap::new(),
|
||||||
unreconciled_worker_writes: Default::default(),
|
unreconciled_worker_writes: Default::default(),
|
||||||
|
subject_delete_worker_fences: BTreeMap::new(),
|
||||||
subject_create_recovery_fences: BTreeMap::new(),
|
subject_create_recovery_fences: BTreeMap::new(),
|
||||||
subject_release_operations: BTreeMap::new(),
|
subject_release_operations: BTreeMap::new(),
|
||||||
workdir_operations: BTreeMap::new(),
|
workdir_operations: BTreeMap::new(),
|
||||||
@@ -5722,6 +5726,7 @@ impl RuntimeState {
|
|||||||
next_diagnostic_id: 1,
|
next_diagnostic_id: 1,
|
||||||
workers: BTreeMap::new(),
|
workers: BTreeMap::new(),
|
||||||
unreconciled_worker_writes: Default::default(),
|
unreconciled_worker_writes: Default::default(),
|
||||||
|
subject_delete_worker_fences: BTreeMap::new(),
|
||||||
subject_create_recovery_fences: BTreeMap::new(),
|
subject_create_recovery_fences: BTreeMap::new(),
|
||||||
subject_release_operations: BTreeMap::new(),
|
subject_release_operations: BTreeMap::new(),
|
||||||
workdir_operations: BTreeMap::new(),
|
workdir_operations: BTreeMap::new(),
|
||||||
@@ -5773,6 +5778,7 @@ impl RuntimeState {
|
|||||||
next_diagnostic_id,
|
next_diagnostic_id,
|
||||||
workers,
|
workers,
|
||||||
unreconciled_worker_writes: Default::default(),
|
unreconciled_worker_writes: Default::default(),
|
||||||
|
subject_delete_worker_fences: persisted.subject_delete_worker_fences,
|
||||||
subject_create_recovery_fences: persisted.subject_create_recovery_fences,
|
subject_create_recovery_fences: persisted.subject_create_recovery_fences,
|
||||||
subject_release_operations: persisted
|
subject_release_operations: persisted
|
||||||
.subject_releases
|
.subject_releases
|
||||||
@@ -5804,6 +5810,7 @@ impl RuntimeState {
|
|||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
fn persisted_state(&self) -> PersistedRuntimeState {
|
fn persisted_state(&self) -> PersistedRuntimeState {
|
||||||
PersistedRuntimeState {
|
PersistedRuntimeState {
|
||||||
|
subject_delete_worker_fences: self.subject_delete_worker_fences.clone(),
|
||||||
subject_create_recovery_fences: self.subject_create_recovery_fences.clone(),
|
subject_create_recovery_fences: self.subject_create_recovery_fences.clone(),
|
||||||
subject_releases: self.subject_release_operations.values().cloned().collect(),
|
subject_releases: self.subject_release_operations.values().cloned().collect(),
|
||||||
display_name: self.display_name.clone(),
|
display_name: self.display_name.clone(),
|
||||||
@@ -6027,7 +6034,9 @@ impl RuntimeState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn ensure_subject_create_not_fenced(&self, worker_id: WorkerId) -> Result<(), RuntimeError> {
|
fn ensure_subject_create_not_fenced(&self, worker_id: WorkerId) -> Result<(), RuntimeError> {
|
||||||
if self.subject_create_recovery_fences.contains_key(&worker_id) {
|
if self.subject_create_recovery_fences.contains_key(&worker_id)
|
||||||
|
|| self.subject_delete_worker_fences.contains_key(&worker_id)
|
||||||
|
{
|
||||||
return Err(RuntimeError::InvalidRequest(
|
return Err(RuntimeError::InvalidRequest(
|
||||||
"Subject create identity is irreversibly fenced".into(),
|
"Subject create identity is irreversibly fenced".into(),
|
||||||
));
|
));
|
||||||
@@ -6040,6 +6049,14 @@ impl RuntimeState {
|
|||||||
worker: &WorkerRef,
|
worker: &WorkerRef,
|
||||||
allow_disconnected: bool,
|
allow_disconnected: bool,
|
||||||
) -> Result<(), RuntimeError> {
|
) -> Result<(), RuntimeError> {
|
||||||
|
if allow_disconnected
|
||||||
|
&& self
|
||||||
|
.subject_delete_worker_fences
|
||||||
|
.get(&worker.worker_id)
|
||||||
|
.is_some_and(|fence| fence.response.is_some())
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
self.ensure_subject_create_not_fenced(worker.worker_id)?;
|
self.ensure_subject_create_not_fenced(worker.worker_id)?;
|
||||||
use runtime_api::SubjectReleaseState as State;
|
use runtime_api::SubjectReleaseState as State;
|
||||||
if self
|
if self
|
||||||
@@ -6110,6 +6127,13 @@ impl RuntimeState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn forget_workspace_owner_if_unused(&mut self, workspace_id: &str) -> bool {
|
fn forget_workspace_owner_if_unused(&mut self, workspace_id: &str) -> bool {
|
||||||
|
if self
|
||||||
|
.subject_delete_worker_fences
|
||||||
|
.values()
|
||||||
|
.any(|fence| fence.request.workspace_id == workspace_id)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if self
|
if self
|
||||||
.subject_create_recovery_fences
|
.subject_create_recovery_fences
|
||||||
.values()
|
.values()
|
||||||
@@ -8767,6 +8791,8 @@ mod tests {
|
|||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct TestExecutionBackend {
|
struct TestExecutionBackend {
|
||||||
|
deletion_confirmed: AtomicBool,
|
||||||
|
deletion_checks: std::sync::atomic::AtomicUsize,
|
||||||
recovery_absent: AtomicBool,
|
recovery_absent: AtomicBool,
|
||||||
recovery_checks: std::sync::atomic::AtomicUsize,
|
recovery_checks: std::sync::atomic::AtomicUsize,
|
||||||
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
||||||
@@ -8848,6 +8874,23 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerExecutionBackend for TestExecutionBackend {
|
impl WorkerExecutionBackend for TestExecutionBackend {
|
||||||
|
fn preflight_subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.deletion_checks.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if self.deletion_confirmed.load(Ordering::SeqCst) {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err("shutdown is unconfirmed".into())
|
||||||
|
}
|
||||||
|
}
|
||||||
fn recover_subject_create(
|
fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
_request: &runtime_api::SubjectCreateRecoveryRequest,
|
_request: &runtime_api::SubjectCreateRecoveryRequest,
|
||||||
@@ -14074,7 +14117,7 @@ mod tests {
|
|||||||
})
|
})
|
||||||
.unwrap_err();
|
.unwrap_err();
|
||||||
assert!(error.to_string().contains(
|
assert!(error.to_string().contains(
|
||||||
"unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, or 14"
|
"unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, 14, or 15"
|
||||||
));
|
));
|
||||||
|
|
||||||
let _ = std::fs::remove_dir_all(root);
|
let _ = std::fs::remove_dir_all(root);
|
||||||
@@ -14259,6 +14302,8 @@ mod tests {
|
|||||||
let _ = std::fs::remove_dir_all(root);
|
let _ = std::fs::remove_dir_all(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
include!("runtime/subject_delete_tests.rs");
|
||||||
|
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
#[test]
|
#[test]
|
||||||
fn subject_create_recovery_is_durable_exact_and_rechecks_absence() {
|
fn subject_create_recovery_is_durable_exact_and_rechecks_absence() {
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Permanent exclusion survives WorkerRemoval. A receipt never substitutes for a
|
||||||
|
/// fresh execution reconciliation on exact retry, including after Runtime reopen.
|
||||||
|
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(crate) struct SubjectDeleteWorkerFence {
|
||||||
|
pub(crate) request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
pub(crate) response: Option<runtime_api::SubjectDeleteWorkerResponse>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
pub fn subject_delete_worker_scoped(
|
||||||
|
&self,
|
||||||
|
scope: &RuntimeWorkspaceScope,
|
||||||
|
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeError> {
|
||||||
|
if request.workspace_id != scope.workspace_id
|
||||||
|
|| request.create_fingerprint.trim().is_empty()
|
||||||
|
|| request.operation_id.trim().is_empty()
|
||||||
|
|| request.create_fingerprint.len() > 512
|
||||||
|
|| request.operation_id.len() > 512
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"invalid Subject delete Worker identity".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let lease = self.worker_operation_lock(request.worker_id)?;
|
||||||
|
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
let worker_ref = WorkerRef::new(request.worker_id);
|
||||||
|
let backend = {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
if state.backend == RuntimeBackendKind::Memory {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject delete requires durable storage".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.ensure_workspace_owner(scope, false)?;
|
||||||
|
if state.workers.contains_key(&request.worker_id) {
|
||||||
|
state.refresh_uncertain_worker_write(&worker_ref)?;
|
||||||
|
let worker = state.worker(&worker_ref)?;
|
||||||
|
if worker.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|
||||||
|
|| worker.request.as_ref().is_none_or(|create| {
|
||||||
|
create.create_fingerprint != request.create_fingerprint
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject delete Worker identity mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(recovery) = state.subject_create_recovery_fences.get(&request.worker_id)
|
||||||
|
&& (recovery.request.workspace_id != request.workspace_id
|
||||||
|
|| recovery.request.create_fingerprint != request.create_fingerprint)
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject recovery identity mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if state.subject_delete_worker_fences.values().any(|fence| {
|
||||||
|
fence.request.workspace_id == request.workspace_id
|
||||||
|
&& fence.request.operation_id == request.operation_id
|
||||||
|
&& fence.request.worker_id != request.worker_id
|
||||||
|
}) {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject delete operation reused for another Worker".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.execution_backend.clone()
|
||||||
|
};
|
||||||
|
// Unpublished execution identity is checked without holding the catalog
|
||||||
|
// mutex. The Worker operation lock excludes Runtime create/restore.
|
||||||
|
if let Some(backend) = backend.as_ref() {
|
||||||
|
backend
|
||||||
|
.preflight_subject_delete_worker(&request)
|
||||||
|
.map_err(RuntimeError::InvalidRequest)?;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_workspace_owner(scope, false)?;
|
||||||
|
if state.subject_delete_worker_fences.values().any(|fence| {
|
||||||
|
fence.request.workspace_id == request.workspace_id
|
||||||
|
&& fence.request.operation_id == request.operation_id
|
||||||
|
&& fence.request.worker_id != request.worker_id
|
||||||
|
}) {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject delete operation reused for another Worker".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if let Some(saved) = state.subject_delete_worker_fences.get(&request.worker_id) {
|
||||||
|
if saved.request != request {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"different Subject delete Worker request retained".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
state.ensure_workspace_owner(scope, true)?;
|
||||||
|
state.subject_delete_worker_fences.insert(
|
||||||
|
request.worker_id,
|
||||||
|
SubjectDeleteWorkerFence {
|
||||||
|
request: request.clone(),
|
||||||
|
response: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Keep exclusion on persistence uncertainty; never start execution
|
||||||
|
// cleanup until this durable write has been confirmed.
|
||||||
|
state.persist_runtime_snapshot()?;
|
||||||
|
}
|
||||||
|
let backend = backend.ok_or_else(|| {
|
||||||
|
RuntimeError::InvalidRequest(
|
||||||
|
"Subject Worker shutdown cannot be confirmed without execution backend".into(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
backend
|
||||||
|
.subject_delete_worker(&request)
|
||||||
|
.map_err(RuntimeError::InvalidRequest)?;
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
if let Some(mut worker) = state.workers.get(&request.worker_id).cloned() {
|
||||||
|
let restore_receipt =
|
||||||
|
RuntimeWorkerRestoreResult::superseded_by_stop(worker.pending_restore);
|
||||||
|
worker.pending_stop = None;
|
||||||
|
worker.pending_restore = None;
|
||||||
|
if let Some(context) = worker.restore_candidate_context.take() {
|
||||||
|
context.discard_candidate();
|
||||||
|
}
|
||||||
|
worker.restore_guard.rotate();
|
||||||
|
worker.restore_guard.finish(restore_receipt);
|
||||||
|
worker.status = WorkerStatus::Stopped;
|
||||||
|
worker.worker_state = None;
|
||||||
|
worker.restore_intent = WorkerRestoreIntent::Explicit;
|
||||||
|
worker.internal_workers.clear();
|
||||||
|
worker.execution_bound = false;
|
||||||
|
state.persist_worker_record(&worker)?;
|
||||||
|
state.workers.insert(request.worker_id, worker);
|
||||||
|
state.publish_worker_upsert(request.worker_id)?;
|
||||||
|
}
|
||||||
|
// This is storage-owner evidence under the same lock that permanently
|
||||||
|
// excludes create/restore, not an inference from catalog projection.
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
let originals_absent = !state.workers.contains_key(&request.worker_id)
|
||||||
|
&& state
|
||||||
|
.fs_store()
|
||||||
|
.ok_or_else(|| {
|
||||||
|
RuntimeError::InvalidRequest(
|
||||||
|
"Subject original absence requires fs storage".into(),
|
||||||
|
)
|
||||||
|
})?
|
||||||
|
.worker_originals_absent(request.worker_id)?;
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
let originals_absent = false;
|
||||||
|
let response = runtime_api::SubjectDeleteWorkerResponse {
|
||||||
|
workspace_id: request.workspace_id.clone(),
|
||||||
|
worker_id: request.worker_id,
|
||||||
|
create_fingerprint: request.create_fingerprint.clone(),
|
||||||
|
operation_id: request.operation_id.clone(),
|
||||||
|
originals_absent,
|
||||||
|
};
|
||||||
|
state
|
||||||
|
.subject_delete_worker_fences
|
||||||
|
.get_mut(&request.worker_id)
|
||||||
|
.expect("delete fence retained under Worker operation lock")
|
||||||
|
.response = Some(response.clone());
|
||||||
|
state.persist_runtime_snapshot()?;
|
||||||
|
Ok(response)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
// Runtime component tests use real filesystem persistence and replace only the
|
||||||
|
// external execution boundary. Actual Controller shutdown is tested by the adapter.
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[test]
|
||||||
|
fn subject_delete_reserved_id_survives_unknown_reopen_and_rejects_identity_changes() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let options =
|
||||||
|
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
|
||||||
|
let scoped = scope("workspace-a", "server-a");
|
||||||
|
let create = scoped_task_request("delayed create", "workspace-a");
|
||||||
|
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: scoped.workspace_id.clone(),
|
||||||
|
worker_id: create.worker_id,
|
||||||
|
create_fingerprint: create.create_fingerprint.clone(),
|
||||||
|
operation_id: "subject-delete-1".into(),
|
||||||
|
};
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id]
|
||||||
|
.response
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.create_worker_scoped(&scoped, create.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(backend.deletion_checks.load(Ordering::SeqCst), 1);
|
||||||
|
drop(runtime);
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.create_worker_scoped(&scoped, create.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||||
|
let receipt = runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
let mut expected = serde_json::to_value(&request).unwrap();
|
||||||
|
expected["originals_absent"] = true.into();
|
||||||
|
assert_eq!(serde_json::to_value(&receipt).unwrap(), expected);
|
||||||
|
for index in 0..4 {
|
||||||
|
let mut changed = request.clone();
|
||||||
|
match index {
|
||||||
|
0 => changed.worker_id = WorkerId::now_v7(),
|
||||||
|
1 => changed.operation_id.push_str("-changed"),
|
||||||
|
2 => changed.create_fingerprint.push_str("-changed"),
|
||||||
|
_ => changed.workspace_id.push_str("-changed"),
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, changed)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scope("workspace-a", "server-b"), request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scope("workspace-b", "server-a"), request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.forget_workspace_owner_if_unused("workspace-a")
|
||||||
|
);
|
||||||
|
drop(runtime);
|
||||||
|
// Even a saved receipt must not manufacture fresh shutdown proof on reopen.
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request)
|
||||||
|
.unwrap(),
|
||||||
|
receipt
|
||||||
|
);
|
||||||
|
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[test]
|
||||||
|
fn subject_delete_existing_stopped_or_active_worker_preserves_record_and_allows_canonical_removal()
|
||||||
|
{
|
||||||
|
for stopped in [false, true] {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||||
|
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
|
||||||
|
.with_runtime_id("test-runtime");
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let scoped = scope("workspace-a", "server-a");
|
||||||
|
let create = scoped_task_request("subject body", "workspace-a");
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker_scoped(&scoped, create.clone())
|
||||||
|
.unwrap();
|
||||||
|
if stopped {
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
}
|
||||||
|
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: scoped.workspace_id.clone(),
|
||||||
|
worker_id: create.worker_id,
|
||||||
|
create_fingerprint: create.create_fingerprint.clone(),
|
||||||
|
operation_id: "delete-body".into(),
|
||||||
|
};
|
||||||
|
let mut mismatch = request.clone();
|
||||||
|
mismatch.create_fingerprint.push_str("-wrong");
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, mismatch)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let mut mismatch = request.clone();
|
||||||
|
mismatch.workspace_id = "workspace-b".into();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scope("workspace-b", "server-b"), mismatch)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.subject_delete_worker_fences
|
||||||
|
.contains_key(&create.worker_id)
|
||||||
|
);
|
||||||
|
// A lost Worker settlement cannot be reported as completed deletion.
|
||||||
|
runtime
|
||||||
|
.test_runtime_store()
|
||||||
|
.fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let receipt = runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert!(!receipt.originals_absent, "a stopped catalog record still has original material");
|
||||||
|
assert_eq!(
|
||||||
|
runtime.worker_detail(&worker.worker_ref).unwrap().status,
|
||||||
|
WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_worker(
|
||||||
|
&worker.worker_ref,
|
||||||
|
runtime.test_restore_request(&worker.worker_ref)
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.create_worker_scoped(&scoped, create.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let record = runtime
|
||||||
|
.test_runtime_store()
|
||||||
|
.read_worker_record(&worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(record.worker_id, worker.worker_ref.worker_id);
|
||||||
|
drop(runtime);
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_worker(
|
||||||
|
&worker.worker_ref,
|
||||||
|
runtime.test_restore_request(&worker.worker_ref)
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request)
|
||||||
|
.unwrap(),
|
||||||
|
receipt
|
||||||
|
);
|
||||||
|
runtime
|
||||||
|
.delete_worker_scoped(&scoped, &worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||||
|
assert!(
|
||||||
|
!runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.forget_workspace_owner_if_unused("workspace-a")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[test]
|
||||||
|
fn subject_delete_unavailable_backend_never_issues_receipt_but_preserves_exclusion() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let options =
|
||||||
|
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
|
||||||
|
let scoped = scope("workspace-a", "server-a");
|
||||||
|
let create = scoped_task_request("reserved without backend", "workspace-a");
|
||||||
|
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: scoped.workspace_id.clone(),
|
||||||
|
worker_id: create.worker_id,
|
||||||
|
create_fingerprint: create.create_fingerprint.clone(),
|
||||||
|
operation_id: "delete-offline".into(),
|
||||||
|
};
|
||||||
|
let runtime = Runtime::with_fs_store(options.clone()).unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id]
|
||||||
|
.response
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
drop(runtime);
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap();
|
||||||
|
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scoped, request)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[test]
|
||||||
|
fn subject_delete_missing_catalog_does_not_claim_residual_originals_absent() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
backend.deletion_confirmed.store(true, Ordering::SeqCst);
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap();
|
||||||
|
let scoped = scope("workspace-a", "server-a");
|
||||||
|
let create = scoped_task_request("unpublished original", "workspace-a");
|
||||||
|
let original = root.path().join("workers").join(create.worker_id.to_string()).join("session/original.jsonl");
|
||||||
|
std::fs::create_dir_all(original.parent().unwrap()).unwrap();
|
||||||
|
std::fs::write(&original, "preserve original text").unwrap();
|
||||||
|
let receipt = runtime.subject_delete_worker_scoped(&scoped, runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: scoped.workspace_id.clone(), worker_id: create.worker_id,
|
||||||
|
create_fingerprint: create.create_fingerprint.clone(), operation_id: "delete-unpublished".into(),
|
||||||
|
}).unwrap();
|
||||||
|
assert!(!receipt.originals_absent);
|
||||||
|
assert_eq!(std::fs::read_to_string(original).unwrap(), "preserve original text");
|
||||||
|
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
|
||||||
|
}
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
// Integration of Runtime's durable fence with the real adapter and Controller;
|
||||||
|
// MockFactory replaces only model/provider/profile construction for a hanging run.
|
||||||
|
#[test]
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn subject_delete_running_controller_confirms_shutdown_and_preserves_history() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let client = MockClient::sequential(vec![
|
||||||
|
MockResponse::Complete(simple_text_events()),
|
||||||
|
MockResponse::Hang(vec![]),
|
||||||
|
]);
|
||||||
|
let backend = Arc::new(
|
||||||
|
WorkerRuntimeExecutionBackend::new(MockFactory {
|
||||||
|
client: client.clone(),
|
||||||
|
runtime_base: root.path().join("control"),
|
||||||
|
cwd: root.path().to_path_buf(),
|
||||||
|
store_dir: root.path().join("sessions"),
|
||||||
|
worker_metadata_dir: root.path().join("metadata"),
|
||||||
|
observed_cwds: Arc::default(),
|
||||||
|
observed_workspace_clients: Arc::default(),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||||
|
crate::fs_store::FsRuntimeStoreOptions::new(root.path().join("catalog"))
|
||||||
|
.with_runtime_id("delete-test"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test");
|
||||||
|
let mut create = create_request("running Subject body");
|
||||||
|
create.workspace_api = Some(WorkspaceApiRef {
|
||||||
|
workspace_id: scope.workspace_id.clone(),
|
||||||
|
base_url: "https://workspace.invalid".into(),
|
||||||
|
});
|
||||||
|
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||||
|
workspace_id: scope.workspace_id.clone(),
|
||||||
|
language: "English".into(),
|
||||||
|
});
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker_scoped(&scope, create.clone())
|
||||||
|
.unwrap();
|
||||||
|
runtime
|
||||||
|
.send_input(&worker.worker_ref, WorkerInput::user("settled history"))
|
||||||
|
.unwrap();
|
||||||
|
let execution = backend.workers.lock().unwrap()[&worker.worker_ref].clone();
|
||||||
|
backend
|
||||||
|
.run_on_adapter_runtime({
|
||||||
|
let handle = execution.handle.clone();
|
||||||
|
let client = client.clone();
|
||||||
|
async move {
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
while client.call_count.load(Ordering::SeqCst) < 1
|
||||||
|
|| handle.shared_state.catalog_status() != WorkerStatus::Idle
|
||||||
|
{
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let before = persisted_files(&root.path().join("sessions"));
|
||||||
|
assert!(!before.is_empty());
|
||||||
|
runtime
|
||||||
|
.send_input(&worker.worker_ref, WorkerInput::user("run until stopped"))
|
||||||
|
.unwrap();
|
||||||
|
backend
|
||||||
|
.run_on_adapter_runtime({
|
||||||
|
let handle = execution.handle.clone();
|
||||||
|
let client = client.clone();
|
||||||
|
async move {
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
while client.call_count.load(Ordering::SeqCst) < 2 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Running);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: scope.workspace_id.clone(),
|
||||||
|
worker_id: create.worker_id,
|
||||||
|
create_fingerprint: create.create_fingerprint.clone(),
|
||||||
|
operation_id: "delete-running".into(),
|
||||||
|
};
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scope, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert!(execution.handle.protocol_is_closed());
|
||||||
|
assert!(execution.tasks.tasks.lock().unwrap().is_empty());
|
||||||
|
assert!(backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(backend.pending_restores.lock().unwrap().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
runtime.worker_detail(&worker.worker_ref).unwrap().status,
|
||||||
|
crate::catalog::WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
// Preserve the settled prefix. Ordinary Controller empty-turn rollback remains unchanged.
|
||||||
|
let after = persisted_files(&root.path().join("sessions"));
|
||||||
|
for (path, bytes) in before {
|
||||||
|
let entries = |bytes: &[u8]| {
|
||||||
|
bytes
|
||||||
|
.split(|b| *b == b'\n')
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
.map(|line| serde_json::from_slice::<serde_json::Value>(line).unwrap())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
entries(&after[&path]).starts_with(&entries(&bytes)),
|
||||||
|
"{path:?} BEFORE={:?} AFTER={:?}",
|
||||||
|
entries(&bytes),
|
||||||
|
entries(&after[&path])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(runtime.create_worker_scoped(&scope, create).is_err());
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_worker(
|
||||||
|
&worker.worker_ref,
|
||||||
|
runtime.test_restore_request(&worker.worker_ref)
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scope, request)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(client.call_count.load(Ordering::SeqCst), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
fn subject_delete_unregistered_profile_controller_retries_unknown_shutdown_without_erasing_session()
|
||||||
|
{
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let (backend, request) = profile_stop_fixture(root.path());
|
||||||
|
let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone();
|
||||||
|
let deletion = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: request
|
||||||
|
.workspace_scope
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.workspace_id
|
||||||
|
.clone(),
|
||||||
|
worker_id: request.worker_ref.worker_id,
|
||||||
|
create_fingerprint: request.request.create_fingerprint.clone(),
|
||||||
|
operation_id: "delete-unregistered".into(),
|
||||||
|
};
|
||||||
|
let mut bad = deletion.clone();
|
||||||
|
bad.create_fingerprint.push_str("-wrong");
|
||||||
|
assert!(backend.preflight_subject_delete_worker(&bad).is_err());
|
||||||
|
assert!(backend.subject_delete_worker(&bad).is_err());
|
||||||
|
let mut bad = deletion.clone();
|
||||||
|
bad.workspace_id.push_str("-wrong");
|
||||||
|
assert!(backend.subject_delete_worker(&bad).is_err());
|
||||||
|
assert!(
|
||||||
|
backend.admissions.lock().unwrap()[&request.worker_ref]
|
||||||
|
.deletion
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let preparing = execution.clone();
|
||||||
|
backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
prepare_rejected_shutdown(&preparing, false, None).await;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let session_root = root
|
||||||
|
.path()
|
||||||
|
.join("runtime/workers")
|
||||||
|
.join(request.worker_ref.worker_id.to_string())
|
||||||
|
.join("session");
|
||||||
|
let before = persisted_files(&session_root);
|
||||||
|
assert!(!before.is_empty());
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.subject_delete_worker(&deletion)
|
||||||
|
.unwrap_err()
|
||||||
|
.contains("Conflict")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.workers
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.contains_key(&request.worker_ref)
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
backend.restore_worker(request.clone()),
|
||||||
|
WorkerExecutionSpawnResult::Rejected(_)
|
||||||
|
));
|
||||||
|
backend.subject_delete_worker(&deletion).unwrap();
|
||||||
|
backend.subject_delete_worker(&deletion).unwrap();
|
||||||
|
assert!(execution.handle.protocol_is_closed());
|
||||||
|
assert!(execution.tasks.tasks.lock().unwrap().is_empty());
|
||||||
|
assert!(backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert_eq!(persisted_files(&session_root), before);
|
||||||
|
assert!(backend.preflight_restore(&request).is_err());
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.activate_restored_worker(request.operation_id, &request.worker_ref)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn subject_delete_pending_restore_fences_late_controller_and_never_restarts_it() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let restores = Arc::new(AtomicUsize::new(0));
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let handles = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
let backend = WorkerRuntimeExecutionBackend::new(LateControllerFactory {
|
||||||
|
inner: MockFactory {
|
||||||
|
client: MockClient::new(vec![]),
|
||||||
|
runtime_base: root.path().join("control"),
|
||||||
|
cwd: root.path().to_path_buf(),
|
||||||
|
store_dir: root.path().join("sessions"),
|
||||||
|
worker_metadata_dir: root.path().join("metadata"),
|
||||||
|
observed_cwds: Arc::default(),
|
||||||
|
observed_workspace_clients: Arc::default(),
|
||||||
|
},
|
||||||
|
restores: restores.clone(),
|
||||||
|
release: release.clone(),
|
||||||
|
actual_handles: handles.clone(),
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.with_spawn_restore_timeout(Duration::from_millis(1));
|
||||||
|
let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7()));
|
||||||
|
request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new(
|
||||||
|
"ws-test",
|
||||||
|
"server-test",
|
||||||
|
));
|
||||||
|
request.request.workspace_api = Some(WorkspaceApiRef {
|
||||||
|
workspace_id: "ws-test".into(),
|
||||||
|
base_url: "https://workspace.invalid".into(),
|
||||||
|
});
|
||||||
|
assert!(matches!(
|
||||||
|
backend.restore_worker(request.clone()),
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired { .. }
|
||||||
|
));
|
||||||
|
let deletion = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: "ws-test".into(),
|
||||||
|
worker_id: request.worker_ref.worker_id,
|
||||||
|
create_fingerprint: request.request.create_fingerprint.clone(),
|
||||||
|
operation_id: "delete-pending".into(),
|
||||||
|
};
|
||||||
|
// Unfinished factory ownership is unknown, never shutdown success.
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.subject_delete_worker(&deletion)
|
||||||
|
.unwrap_err()
|
||||||
|
.contains("pending")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.pending_restores
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.contains_key(&request.worker_ref)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.activate_restored_worker(request.operation_id, &request.worker_ref)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
release.notify_one();
|
||||||
|
backend.subject_delete_worker(&deletion).unwrap();
|
||||||
|
assert!(backend.pending_restores.lock().unwrap().is_empty());
|
||||||
|
assert!(backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert_eq!(restores.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(handles.lock().unwrap().len(), 1);
|
||||||
|
assert!(handles.lock().unwrap()[0].protocol_is_closed());
|
||||||
|
assert!(matches!(
|
||||||
|
backend.restore_worker(request),
|
||||||
|
WorkerExecutionSpawnResult::Rejected(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
// Actual create factory future remains behind Runtime's Worker operation lock;
|
||||||
|
// deletion must wait for that owner, then permanently reject the delayed retry.
|
||||||
|
struct SubjectDeleteDeferredCreate {
|
||||||
|
entered: mpsc::Sender<()>,
|
||||||
|
release: Arc<tokio::sync::Notify>,
|
||||||
|
}
|
||||||
|
#[async_trait]
|
||||||
|
impl RuntimeWorkerFactory for SubjectDeleteDeferredCreate {
|
||||||
|
async fn prepare_restore(
|
||||||
|
&self,
|
||||||
|
_: WorkerExecutionRestoreRequest,
|
||||||
|
) -> Result<Box<dyn RuntimeRestoreCandidate>, RuntimeRestoreFailure> {
|
||||||
|
panic!("fenced create must never restore")
|
||||||
|
}
|
||||||
|
async fn spawn_controller(
|
||||||
|
&self,
|
||||||
|
_: WorkerExecutionSpawnRequest,
|
||||||
|
) -> Result<RuntimeWorkerController, String> {
|
||||||
|
self.entered.send(()).unwrap();
|
||||||
|
self.release.notified().await;
|
||||||
|
Err("create failed before any Controller".into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_delete_serializes_inflight_create_and_rejects_all_late_admission() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let (entered, entry) = mpsc::channel();
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let backend = Arc::new(
|
||||||
|
WorkerRuntimeExecutionBackend::new(SubjectDeleteDeferredCreate {
|
||||||
|
entered,
|
||||||
|
release: release.clone(),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||||
|
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("delete-test"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test");
|
||||||
|
let mut create = create_request("inflight create");
|
||||||
|
create.workspace_api = Some(WorkspaceApiRef {
|
||||||
|
workspace_id: scope.workspace_id.clone(),
|
||||||
|
base_url: "https://workspace.invalid".into(),
|
||||||
|
});
|
||||||
|
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||||
|
workspace_id: scope.workspace_id.clone(),
|
||||||
|
language: "English".into(),
|
||||||
|
});
|
||||||
|
let creating = {
|
||||||
|
let runtime = runtime.clone();
|
||||||
|
let scope = scope.clone();
|
||||||
|
let create = create.clone();
|
||||||
|
std::thread::spawn(move || runtime.create_worker_scoped(&scope, create))
|
||||||
|
};
|
||||||
|
entry.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: scope.workspace_id.clone(),
|
||||||
|
worker_id: create.worker_id,
|
||||||
|
create_fingerprint: create.create_fingerprint.clone(),
|
||||||
|
operation_id: "delete-creating".into(),
|
||||||
|
};
|
||||||
|
let deleting = {
|
||||||
|
let runtime = runtime.clone();
|
||||||
|
let scope = scope.clone();
|
||||||
|
let request = request.clone();
|
||||||
|
std::thread::spawn(move || runtime.subject_delete_worker_scoped(&scope, request))
|
||||||
|
};
|
||||||
|
release.notify_one();
|
||||||
|
assert!(creating.join().unwrap().is_err());
|
||||||
|
deleting.join().unwrap().unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.create_worker_scoped(&scope, create.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(backend.workers.lock().unwrap().is_empty());
|
||||||
|
let worker_ref = WorkerRef::new(create.worker_id);
|
||||||
|
let spawn = WorkerExecutionSpawnRequest {
|
||||||
|
worker_ref: worker_ref.clone(),
|
||||||
|
request: create.clone(),
|
||||||
|
workspace_scope: Some(scope.clone()),
|
||||||
|
context: test_execution_context(worker_ref.clone()),
|
||||||
|
workdir_attachments: BTreeMap::new(),
|
||||||
|
config_bundle: None,
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
backend.spawn_worker(spawn),
|
||||||
|
WorkerExecutionSpawnResult::Rejected(_)
|
||||||
|
));
|
||||||
|
let mut restore = direct_restore_request(worker_ref);
|
||||||
|
restore.request = create;
|
||||||
|
restore.workspace_scope = Some(scope);
|
||||||
|
assert!(matches!(
|
||||||
|
backend.restore_worker(restore),
|
||||||
|
WorkerExecutionSpawnResult::Rejected(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
#[test]
|
||||||
|
fn subject_delete_schema14_migration_adds_exclusion_store_without_changing_worker_or_receipts() {
|
||||||
|
for with_fence in [false, true] {
|
||||||
|
let (_temp, store, worker, path) = canonical_worker_store();
|
||||||
|
let root = store.root.clone();
|
||||||
|
let mut legacy =
|
||||||
|
serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap();
|
||||||
|
legacy["schema_version"] = serde_json::json!(RESTORE_REPAIR_SCHEMA_VERSION);
|
||||||
|
fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap();
|
||||||
|
let id = WorkerId::now_v7();
|
||||||
|
let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id,
|
||||||
|
"create_fingerprint": "fenced-create", "operation_id": "delete-1"});
|
||||||
|
let mut response = request.clone();
|
||||||
|
response["originals_absent"] = false.into();
|
||||||
|
let fence =
|
||||||
|
serde_json::json!({ id.to_string(): {"request": request, "response": response} });
|
||||||
|
let mut runtime = serde_json::json!({"schema_version": RESTORE_REPAIR_SCHEMA_VERSION,
|
||||||
|
"display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
|
||||||
|
"workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [],
|
||||||
|
"config_bundles": {}, "subject_create_recovery_fences": {}, "subject_releases": []});
|
||||||
|
if with_fence {
|
||||||
|
runtime["subject_delete_worker_fences"] = fence.clone();
|
||||||
|
}
|
||||||
|
fs::write(
|
||||||
|
root.join(RUNTIME_FILE),
|
||||||
|
serde_json::to_vec(&runtime).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drop(store);
|
||||||
|
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("test-runtime");
|
||||||
|
let before = fs::read(&path).unwrap();
|
||||||
|
assert!(
|
||||||
|
FsRuntimeStore::migration_plan(&options)
|
||||||
|
.unwrap()
|
||||||
|
.migration_required
|
||||||
|
);
|
||||||
|
assert_eq!(fs::read(&path).unwrap(), before);
|
||||||
|
let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
|
||||||
|
legacy["schema_version"] = serde_json::json!(SCHEMA_VERSION);
|
||||||
|
assert_eq!(
|
||||||
|
read_json::<serde_json::Value>(&path, "read migrated Worker").unwrap(),
|
||||||
|
legacy
|
||||||
|
);
|
||||||
|
let expected = if with_fence {
|
||||||
|
fence
|
||||||
|
} else {
|
||||||
|
serde_json::json!({})
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&opened.state.as_ref().unwrap().subject_delete_worker_fences)
|
||||||
|
.unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
opened
|
||||||
|
.store
|
||||||
|
.write_runtime_snapshot(opened.state.as_ref().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
drop(opened);
|
||||||
|
let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&reopened.state.unwrap().subject_delete_worker_fences).unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_delete_store_rejects_missing_exclusion_field_malformed_receipt_and_workspace_owner() {
|
||||||
|
for invalid in 0..4 {
|
||||||
|
let (_temp, store, _worker, _path) = canonical_worker_store();
|
||||||
|
let root = store.root.clone();
|
||||||
|
let id = WorkerId::now_v7();
|
||||||
|
let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id,
|
||||||
|
"create_fingerprint": "fenced-create", "operation_id": "delete-1"});
|
||||||
|
let mut response = request.clone();
|
||||||
|
response["originals_absent"] = false.into();
|
||||||
|
let mut runtime = serde_json::json!({"schema_version": SCHEMA_VERSION,
|
||||||
|
"display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
|
||||||
|
"workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [],
|
||||||
|
"subject_delete_worker_fences": { id.to_string(): {"request": request, "response": response} }});
|
||||||
|
match invalid {
|
||||||
|
0 => {
|
||||||
|
runtime
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("subject_delete_worker_fences");
|
||||||
|
}
|
||||||
|
1 => {
|
||||||
|
runtime["subject_delete_worker_fences"][id.to_string()]["response"]["create_fingerprint"] =
|
||||||
|
"different".into()
|
||||||
|
}
|
||||||
|
2 => runtime["workspace_owners"] = serde_json::json!({}),
|
||||||
|
_ => {
|
||||||
|
runtime["subject_delete_worker_fences"][id.to_string()]["request"]["worker_id"] =
|
||||||
|
WorkerId::now_v7().to_string().into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let bytes = serde_json::to_vec(&runtime).unwrap();
|
||||||
|
fs::write(root.join(RUNTIME_FILE), &bytes).unwrap();
|
||||||
|
drop(store);
|
||||||
|
assert!(FsRuntimeStore::open_or_create(root.clone(), "test-runtime").is_err());
|
||||||
|
assert_eq!(fs::read(root.join(RUNTIME_FILE)).unwrap(), bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -215,7 +215,8 @@ impl StoppedWorkerTemplate {
|
|||||||
snapshot
|
snapshot
|
||||||
.validate(&root.join(RUNTIME_FILE))
|
.validate(&root.join(RUNTIME_FILE))
|
||||||
.map_err(|_| reject("unsupported Runtime schema"))?;
|
.map_err(|_| reject("unsupported Runtime schema"))?;
|
||||||
if !snapshot.subject_create_recovery_fences.is_empty()
|
if !snapshot.subject_delete_worker_fences.is_empty()
|
||||||
|
|| !snapshot.subject_create_recovery_fences.is_empty()
|
||||||
|| !snapshot.subject_releases.is_empty()
|
|| !snapshot.subject_releases.is_empty()
|
||||||
{
|
{
|
||||||
return Err(reject("source has lifecycle operations"));
|
return Err(reject("source has lifecycle operations"));
|
||||||
@@ -427,6 +428,7 @@ impl StoppedWorkerTemplate {
|
|||||||
)?;
|
)?;
|
||||||
self.validate_session(&session)?;
|
self.validate_session(&session)?;
|
||||||
store.write_runtime_snapshot(&PersistedRuntimeState {
|
store.write_runtime_snapshot(&PersistedRuntimeState {
|
||||||
|
subject_delete_worker_fences: BTreeMap::new(),
|
||||||
subject_create_recovery_fences: BTreeMap::new(),
|
subject_create_recovery_fences: BTreeMap::new(),
|
||||||
subject_releases: Vec::new(),
|
subject_releases: Vec::new(),
|
||||||
// Match normal store initialization: this is catalog availability,
|
// Match normal store initialization: this is catalog availability,
|
||||||
|
|||||||
@@ -3415,6 +3415,14 @@ struct SubjectControllerReservation {
|
|||||||
state: runtime_api::SubjectReleaseState,
|
state: runtime_api::SubjectReleaseState,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Admission identity is retained even when execution publication or stop settlement
|
||||||
|
// failed. Runtime owns durable exclusion; this guards the adapter's delayed calls.
|
||||||
|
struct WorkerAdmission {
|
||||||
|
workspace_id: Option<String>,
|
||||||
|
create_fingerprint: String,
|
||||||
|
deletion: Option<runtime_api::SubjectDeleteWorkerRequest>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Worker execution backend backed by real worker crate Workers.
|
/// Worker execution backend backed by real worker crate Workers.
|
||||||
pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -3438,6 +3446,7 @@ pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
|||||||
>,
|
>,
|
||||||
>,
|
>,
|
||||||
worker_locks: Mutex<HashMap<WorkerRef, Arc<Mutex<()>>>>,
|
worker_locks: Mutex<HashMap<WorkerRef, Arc<Mutex<()>>>>,
|
||||||
|
admissions: Mutex<HashMap<WorkerRef, WorkerAdmission>>,
|
||||||
spawn_restore_timeout: Duration,
|
spawn_restore_timeout: Duration,
|
||||||
subject_reservations: Mutex<HashMap<(WorkerRef, String), SubjectControllerReservation>>,
|
subject_reservations: Mutex<HashMap<(WorkerRef, String), SubjectControllerReservation>>,
|
||||||
shutdown_admission_timeout: Duration,
|
shutdown_admission_timeout: Duration,
|
||||||
@@ -3464,12 +3473,92 @@ where
|
|||||||
pending_restores: Mutex::new(HashMap::new()),
|
pending_restores: Mutex::new(HashMap::new()),
|
||||||
completed_restore_failures: Mutex::new(HashMap::new()),
|
completed_restore_failures: Mutex::new(HashMap::new()),
|
||||||
worker_locks: Mutex::new(HashMap::new()),
|
worker_locks: Mutex::new(HashMap::new()),
|
||||||
|
admissions: Mutex::new(HashMap::new()),
|
||||||
spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT,
|
spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT,
|
||||||
subject_reservations: Mutex::new(HashMap::new()),
|
subject_reservations: Mutex::new(HashMap::new()),
|
||||||
shutdown_admission_timeout: Duration::from_secs(5),
|
shutdown_admission_timeout: Duration::from_secs(5),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Caller holds the adapter Worker lock.
|
||||||
|
fn admit_worker(
|
||||||
|
&self,
|
||||||
|
worker: &WorkerRef,
|
||||||
|
workspace: Option<&str>,
|
||||||
|
fingerprint: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let mut admissions = self
|
||||||
|
.admissions
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "worker admission lock poisoned")?;
|
||||||
|
if let Some(saved) = admissions.get(worker) {
|
||||||
|
if saved.deletion.is_some() {
|
||||||
|
return Err("Subject Worker permanently excluded".into());
|
||||||
|
}
|
||||||
|
if saved.workspace_id.as_deref() != workspace || saved.create_fingerprint != fingerprint
|
||||||
|
{
|
||||||
|
return Err("Worker admission identity mismatch".into());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
admissions.insert(
|
||||||
|
worker.clone(),
|
||||||
|
WorkerAdmission {
|
||||||
|
workspace_id: workspace.map(str::to_string),
|
||||||
|
create_fingerprint: fingerprint.into(),
|
||||||
|
deletion: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_subject_delete_identity(
|
||||||
|
&self,
|
||||||
|
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let admissions = self
|
||||||
|
.admissions
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "worker admission lock poisoned")?;
|
||||||
|
if let Some(saved) = admissions.get(&WorkerRef::new(request.worker_id)) {
|
||||||
|
if saved.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|
||||||
|
|| saved.create_fingerprint != request.create_fingerprint
|
||||||
|
|| saved
|
||||||
|
.deletion
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|saved| saved != request)
|
||||||
|
{
|
||||||
|
return Err("Subject delete execution identity mismatch".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(pending) = self.pending_restore(&WorkerRef::new(request.worker_id))? {
|
||||||
|
if pending
|
||||||
|
.request
|
||||||
|
.workspace_scope
|
||||||
|
.as_ref()
|
||||||
|
.map(|scope| scope.workspace_id.as_str())
|
||||||
|
!= Some(request.workspace_id.as_str())
|
||||||
|
|| pending.request.request.create_fingerprint != request.create_fingerprint
|
||||||
|
{
|
||||||
|
return Err("Subject delete pending execution identity mismatch".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_not_subject_deleted(&self, worker: &WorkerRef) -> Result<(), String> {
|
||||||
|
if self
|
||||||
|
.admissions
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "worker admission lock poisoned")?
|
||||||
|
.get(worker)
|
||||||
|
.is_some_and(|admission| admission.deletion.is_some())
|
||||||
|
{
|
||||||
|
return Err("Subject Worker permanently excluded".into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn with_backend_id(mut self, backend_id: impl Into<String>) -> Self {
|
pub fn with_backend_id(mut self, backend_id: impl Into<String>) -> Self {
|
||||||
self.backend_id = backend_id.into();
|
self.backend_id = backend_id.into();
|
||||||
self
|
self
|
||||||
@@ -4739,6 +4828,58 @@ where
|
|||||||
&self.backend_id
|
&self.backend_id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn preflight_subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let worker = WorkerRef::new(request.worker_id);
|
||||||
|
let lock = self.worker_lock(&worker)?;
|
||||||
|
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
|
||||||
|
self.validate_subject_delete_identity(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let worker = WorkerRef::new(request.worker_id);
|
||||||
|
let lock = self.worker_lock(&worker)?;
|
||||||
|
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
|
||||||
|
self.validate_subject_delete_identity(request)?;
|
||||||
|
self.admissions
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "worker admission lock poisoned")?
|
||||||
|
.insert(
|
||||||
|
worker.clone(),
|
||||||
|
WorkerAdmission {
|
||||||
|
workspace_id: Some(request.workspace_id.clone()),
|
||||||
|
create_fingerprint: request.create_fingerprint.clone(),
|
||||||
|
deletion: Some(request.clone()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let result = self.stop_worker_under_lock(&worker);
|
||||||
|
if !result.is_accepted() {
|
||||||
|
return Err(result.message_or_default());
|
||||||
|
}
|
||||||
|
// A live Controller and failed unpublished candidates can coexist. The
|
||||||
|
// ordinary stop's live branch joins the Controller; also reconcile any
|
||||||
|
// factory-owned cache/artifacts before issuing deletion proof.
|
||||||
|
let factory = self.factory.clone();
|
||||||
|
let stopped = worker.clone();
|
||||||
|
self.run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move {
|
||||||
|
factory.reconcile_stopped_worker(&stopped).await
|
||||||
|
})?;
|
||||||
|
self.subject_reservations
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "Subject reservations poisoned")?
|
||||||
|
.retain(|(saved, _), _| saved != &worker);
|
||||||
|
self.completed_restore_failures
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "completed restores poisoned")?
|
||||||
|
.remove(&worker);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn recover_subject_create(
|
fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
request: &runtime_api::SubjectCreateRecoveryRequest,
|
request: &runtime_api::SubjectCreateRecoveryRequest,
|
||||||
@@ -5178,6 +5319,19 @@ where
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if let Err(message) = self.admit_worker(
|
||||||
|
&request.worker_ref,
|
||||||
|
request
|
||||||
|
.workspace_scope
|
||||||
|
.as_ref()
|
||||||
|
.map(|scope| scope.workspace_id.as_str()),
|
||||||
|
&request.request.create_fingerprint,
|
||||||
|
) {
|
||||||
|
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
|
||||||
|
WorkerExecutionOperation::Spawn,
|
||||||
|
message,
|
||||||
|
));
|
||||||
|
}
|
||||||
match self.pending_restore(&request.worker_ref) {
|
match self.pending_restore(&request.worker_ref) {
|
||||||
Ok(None) => {}
|
Ok(None) => {}
|
||||||
Ok(Some(_)) => {
|
Ok(Some(_)) => {
|
||||||
@@ -5405,6 +5559,10 @@ where
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
self.ensure_not_subject_deleted(&request.worker_ref)
|
||||||
|
.map_err(|message| {
|
||||||
|
WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message)
|
||||||
|
})?;
|
||||||
if self
|
if self
|
||||||
.pending_restore(&request.worker_ref)
|
.pending_restore(&request.worker_ref)
|
||||||
.map_err(|message| {
|
.map_err(|message| {
|
||||||
@@ -5499,6 +5657,19 @@ where
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if let Err(message) = self.admit_worker(
|
||||||
|
&request.worker_ref,
|
||||||
|
request
|
||||||
|
.workspace_scope
|
||||||
|
.as_ref()
|
||||||
|
.map(|scope| scope.workspace_id.as_str()),
|
||||||
|
&request.request.create_fingerprint,
|
||||||
|
) {
|
||||||
|
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
|
||||||
|
WorkerExecutionOperation::Restore,
|
||||||
|
message,
|
||||||
|
));
|
||||||
|
}
|
||||||
if let Some(result) =
|
if let Some(result) =
|
||||||
self.completed_restore_failure(request.operation_id, &request.worker_ref)
|
self.completed_restore_failure(request.operation_id, &request.worker_ref)
|
||||||
{
|
{
|
||||||
@@ -6182,6 +6353,7 @@ where
|
|||||||
return Err(message);
|
return Err(message);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
self.ensure_not_subject_deleted(worker_ref)?;
|
||||||
if let Some(pending) = self.pending_restore(worker_ref)? {
|
if let Some(pending) = self.pending_restore(worker_ref)? {
|
||||||
if pending.request.operation_id != operation_id {
|
if pending.request.operation_id != operation_id {
|
||||||
return Err("restored Worker operation identity does not match".into());
|
return Err("restored Worker operation identity does not match".into());
|
||||||
@@ -6820,6 +6992,10 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use worker::PreparedWorker;
|
use worker::PreparedWorker;
|
||||||
|
|
||||||
|
include!("subject_delete_backend_tests.rs");
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
include!("subject_delete_create_tests.rs");
|
||||||
|
|
||||||
// Both candidate and failure suites need the same guarded, checkout-owned
|
// Both candidate and failure suites need the same guarded, checkout-owned
|
||||||
// temporary root, independently of Runtime's optional filesystem catalog.
|
// temporary root, independently of Runtime's optional filesystem catalog.
|
||||||
fn restore_failure_tempdir() -> tempfile::TempDir {
|
fn restore_failure_tempdir() -> tempfile::TempDir {
|
||||||
|
|||||||
@@ -478,7 +478,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
|||||||
(101, "Durable explicit Subject body release".into()),
|
(101, "Durable explicit Subject body release".into()),
|
||||||
(102, "Durable failed Subject creation recovery".into()),
|
(102, "Durable failed Subject creation recovery".into()),
|
||||||
(103, "Passkey metadata and atomic addition results".into()),
|
(103, "Passkey metadata and atomic addition results".into()),
|
||||||
(104, "Converge Passkey results and Job input retention".into())
|
(104, "Converge Passkey results and Job input retention".into()),
|
||||||
|
(105, "Durable Subject deletion snapshots and Job admission fences".into())
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||||
|
|||||||
@@ -1292,6 +1292,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||||
|
Err("Subject deletion fencing is unavailable on this Runtime".into())
|
||||||
|
}
|
||||||
|
|
||||||
fn recover_subject_create(
|
fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
_request: runtime_api::SubjectCreateRecoveryRequest,
|
_request: runtime_api::SubjectCreateRecoveryRequest,
|
||||||
@@ -2269,6 +2276,21 @@ impl RuntimeRegistry {
|
|||||||
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
runtime_id: &str,
|
||||||
|
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeRegistryError> {
|
||||||
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
|
self.runtime(runtime_id)?
|
||||||
|
.subject_delete_worker(request)
|
||||||
|
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
|
||||||
|
runtime_id: runtime_id.into(),
|
||||||
|
code: "subject_delete_worker_unproven".into(),
|
||||||
|
message,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn recover_subject_create(
|
pub fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
runtime_id: &str,
|
||||||
@@ -3367,6 +3389,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||||
|
if request.workspace_id != self.workspace_id {
|
||||||
|
return Err("Subject deletion Workspace mismatch".into());
|
||||||
|
}
|
||||||
|
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
|
||||||
|
self.runtime
|
||||||
|
.subject_delete_worker_scoped(&scope, request)
|
||||||
|
.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn recover_subject_create(
|
fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||||
@@ -4255,7 +4290,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
|
|||||||
}
|
}
|
||||||
if path.contains("/retention/")
|
if path.contains("/retention/")
|
||||||
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|
||||||
|| (path == "/v1/subject-create-recovery" && method == "POST")
|
|| (matches!(
|
||||||
|
path,
|
||||||
|
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
|
||||||
|
) && method == "POST")
|
||||||
{
|
{
|
||||||
return "workers:delete";
|
return "workers:delete";
|
||||||
}
|
}
|
||||||
@@ -5656,6 +5694,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||||
|
self.run_runtime_api(
|
||||||
|
self.request_timeout,
|
||||||
|
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||||
|
move |client| async move { client.subject_delete_worker(request).await },
|
||||||
|
)
|
||||||
|
.map_err(|diagnostic| diagnostic.message)
|
||||||
|
}
|
||||||
|
|
||||||
fn recover_subject_create(
|
fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||||
@@ -8507,6 +8557,57 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remote_subject_delete_uses_generated_exact_identity_transport() {
|
||||||
|
assert_eq!(
|
||||||
|
workspace_runtime_operation("POST", "/v1/subject-delete-worker"),
|
||||||
|
"workers:delete"
|
||||||
|
);
|
||||||
|
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: "workspace-test".into(),
|
||||||
|
worker_id: EmbeddedWorkerId::now_v7(),
|
||||||
|
create_fingerprint: "sha256:reserved-create".into(),
|
||||||
|
operation_id: "delete-worker-1".into(),
|
||||||
|
};
|
||||||
|
let response = runtime_api::SubjectDeleteWorkerResponse {
|
||||||
|
originals_absent: true,
|
||||||
|
workspace_id: request.workspace_id.clone(),
|
||||||
|
worker_id: request.worker_id,
|
||||||
|
create_fingerprint: request.create_fingerprint.clone(),
|
||||||
|
operation_id: request.operation_id.clone(),
|
||||||
|
};
|
||||||
|
// Only remote transport is substituted; actual Controller shutdown and
|
||||||
|
// signed admission are covered by worker-runtime's independent suites.
|
||||||
|
let (base_url, server) = serve_mock_http(vec![mock_response(
|
||||||
|
"POST",
|
||||||
|
"/v1/subject-delete-worker",
|
||||||
|
true,
|
||||||
|
200,
|
||||||
|
serde_json::to_string(&response).unwrap(),
|
||||||
|
)]);
|
||||||
|
let registry = RuntimeRegistry::new(Vec::new());
|
||||||
|
registry.register(
|
||||||
|
RemoteWorkerRuntime::new(
|
||||||
|
RemoteRuntimeConfig::new(
|
||||||
|
"remote:delete",
|
||||||
|
"Deletion transport fixture",
|
||||||
|
base_url,
|
||||||
|
Some("secret-token".into()),
|
||||||
|
),
|
||||||
|
"workspace-test".into(),
|
||||||
|
"http://127.0.0.1:8787".into(),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
registry
|
||||||
|
.subject_delete_worker("remote:delete", request)
|
||||||
|
.unwrap(),
|
||||||
|
response
|
||||||
|
);
|
||||||
|
server.join().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn remote_subject_create_recovery_uses_generated_management_transport() {
|
fn remote_subject_create_recovery_uses_generated_management_transport() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -367,14 +367,54 @@ async fn implementation_result_rejects_forged_evidence_and_rolls_back_without_no
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
// Preserve seeded assignment evidence while rebuilding the actual pre-business
|
||||||
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
|
// capture composition. Changing only a receipt leaves schema105 admission fences
|
||||||
let (dir, store) = tracking_store_fixture().await;
|
// and erasure guards in an otherwise legacy fixture.
|
||||||
tracking_store_assign(&store, "delegation", "assign");
|
fn reconstruct_implementation_schema96(conn: &Connection) -> Result<()> {
|
||||||
let worker = routing_worker_assign(&store, "implementation").worker;
|
assert_eq!(current_schema_version(conn)?, 105);
|
||||||
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
|
let directory = tempfile::tempdir().unwrap();
|
||||||
store.with_conn(|conn| {
|
let path = directory.path().join("retained96.db");
|
||||||
conn.execute_batch("DROP TRIGGER notification_mr_event;
|
prepare_retained_schema(&path, 96);
|
||||||
|
let retained = Connection::open(path)?;
|
||||||
|
for trigger in [
|
||||||
|
"notification_sources_no_delete",
|
||||||
|
"notification_decisions_no_delete",
|
||||||
|
"notification_requests_no_delete",
|
||||||
|
"notification_acknowledgements_no_delete",
|
||||||
|
"notification_failures_no_delete",
|
||||||
|
"notification_job_targets_no_delete",
|
||||||
|
"notification_job_legacy_no_delete",
|
||||||
|
"notification_source_preparation_failures_no_delete",
|
||||||
|
] {
|
||||||
|
let definition: String = retained.query_row(
|
||||||
|
"SELECT sql FROM sqlite_schema WHERE type='trigger' AND name=?1",
|
||||||
|
[trigger],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
conn.execute_batch(&format!("DROP TRIGGER {trigger}; {definition};"))?;
|
||||||
|
}
|
||||||
|
for trigger in [
|
||||||
|
"subject_deleted_job_insert",
|
||||||
|
"subject_deleted_job_result",
|
||||||
|
"subject_deleted_attempt_insert",
|
||||||
|
"subject_deleted_attempt_result",
|
||||||
|
"subject_deleted_worker_reservation",
|
||||||
|
"subject_deleted_notification_source",
|
||||||
|
"subject_deleted_notification_payload",
|
||||||
|
"subject_deleted_notification_send",
|
||||||
|
] {
|
||||||
|
conn.execute_batch(&format!("DROP TRIGGER {trigger};"))?;
|
||||||
|
}
|
||||||
|
// Children before parents; dropping the tables also drops snapshot guards.
|
||||||
|
conn.execute_batch(
|
||||||
|
"DROP TABLE subject_deletion_allocations;
|
||||||
|
DROP TABLE subject_deletion_jobs;
|
||||||
|
DROP TABLE subject_deletion_workers;
|
||||||
|
DROP TABLE subject_deletions;
|
||||||
|
UPDATE __yoi_schema_migrations SET version=104 WHERE version=105;",
|
||||||
|
)?;
|
||||||
|
conn.execute_batch(
|
||||||
|
"DROP TRIGGER notification_mr_event;
|
||||||
DROP TRIGGER notification_mr_source;
|
DROP TRIGGER notification_mr_source;
|
||||||
DROP TRIGGER notification_implementation_result;
|
DROP TRIGGER notification_implementation_result;
|
||||||
DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||||
@@ -396,7 +436,43 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
|||||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;",
|
||||||
|
)?;
|
||||||
|
assert_eq!(current_schema_version(conn)?, 96);
|
||||||
|
assert_eq!(conn.query_row(
|
||||||
|
"SELECT count(*) FROM sqlite_schema WHERE name LIKE 'subject_delet%' OR sql LIKE '%subject_deletion%'",
|
||||||
|
[], |row| row.get::<_, i64>(0),
|
||||||
|
)?, 0);
|
||||||
|
for table in [
|
||||||
|
"notification_sources",
|
||||||
|
"notification_decisions",
|
||||||
|
"notification_requests",
|
||||||
|
"notification_acknowledgements",
|
||||||
|
"notification_failures",
|
||||||
|
"notification_job_targets",
|
||||||
|
"notification_job_legacy_receipts",
|
||||||
|
"notification_source_preparation_failures",
|
||||||
|
"backend_jobs",
|
||||||
|
"backend_job_attempts",
|
||||||
|
"worker_create_reservations",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
state_contract_schema_signature(conn, table),
|
||||||
|
state_contract_schema_signature(&retained, table),
|
||||||
|
"schema96 {table}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
|
||||||
|
let (dir, store) = tracking_store_fixture().await;
|
||||||
|
tracking_store_assign(&store, "delegation", "assign");
|
||||||
|
let worker = routing_worker_assign(&store, "implementation").worker;
|
||||||
|
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
|
||||||
|
store.with_conn(|conn| {
|
||||||
|
reconstruct_implementation_schema96(conn)?;
|
||||||
migrate_business_notifications_v96_to_v97(conn)?;
|
migrate_business_notifications_v96_to_v97(conn)?;
|
||||||
assert_eq!(current_schema_version(conn)?, 97);
|
assert_eq!(current_schema_version(conn)?, 97);
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -522,24 +598,7 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
|||||||
store.with_conn(|conn| {
|
store.with_conn(|conn| {
|
||||||
conn.execute("INSERT INTO ticket_assignment_operation_fences VALUES('w','operation-known-implementation','delegation','r','worker','workspace-orchestrator','tracking-binding',?1)",[serde_json::to_string(&tracking_store_source()).unwrap()])?;
|
conn.execute("INSERT INTO ticket_assignment_operation_fences VALUES('w','operation-known-implementation','delegation','r','worker','workspace-orchestrator','tracking-binding',?1)",[serde_json::to_string(&tracking_store_source()).unwrap()])?;
|
||||||
// Reconstruct the prior composition before its new business capture existed.
|
// Reconstruct the prior composition before its new business capture existed.
|
||||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
reconstruct_implementation_schema96(conn)?;
|
||||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
|
||||||
DROP TRIGGER passkey_metadata_insert;
|
|
||||||
DROP TABLE passkey_addition_receipts;
|
|
||||||
DROP TABLE passkey_metadata;
|
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
|
||||||
DROP TABLE subject_create_recoveries;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
|
||||||
DROP TABLE subject_worker_releases;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
|
||||||
DROP TABLE user_activations;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
|
||||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
|
||||||
conn.execute_batch("INSERT INTO merge_requests VALUES('w','historical-mr','main','open','refs/heads/work','refs/heads/main','3','3');
|
conn.execute_batch("INSERT INTO merge_requests VALUES('w','historical-mr','main','open','refs/heads/work','refs/heads/main','3','3');
|
||||||
INSERT INTO merge_request_ticket_relations VALUES('w','historical-mr','tracking','implements','3');
|
INSERT INTO merge_request_ticket_relations VALUES('w','historical-mr','tracking','implements','3');
|
||||||
INSERT INTO merge_request_thread_events VALUES('w','historical-mr','approved-event',1,'review','{\"subject_ref\":\"source-a\",\"decision\":\"approve\"}',NULL,'3');
|
INSERT INTO merge_request_thread_events VALUES('w','historical-mr','approved-event',1,'review','{\"subject_ref\":\"source-a\",\"decision\":\"approve\"}',NULL,'3');
|
||||||
@@ -691,24 +750,7 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
|||||||
];
|
];
|
||||||
store.with_conn(|conn| {
|
store.with_conn(|conn| {
|
||||||
if historical {
|
if historical {
|
||||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
reconstruct_implementation_schema96(conn)?;
|
||||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
|
||||||
DROP TRIGGER passkey_metadata_insert;
|
|
||||||
DROP TABLE passkey_addition_receipts;
|
|
||||||
DROP TABLE passkey_metadata;
|
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
|
||||||
DROP TABLE subject_create_recoveries;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
|
||||||
DROP TABLE subject_worker_releases;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
|
||||||
DROP TABLE user_activations;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
|
||||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
|
||||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
|
||||||
}
|
}
|
||||||
conn.execute_batch("INSERT INTO merge_requests VALUES('w','actor-mr','main','open','work','main','3','3');
|
conn.execute_batch("INSERT INTO merge_requests VALUES('w','actor-mr','main','open','work','main','3','3');
|
||||||
INSERT INTO merge_request_ticket_relations VALUES('w','actor-mr','tracking','implements','3');")?;
|
INSERT INTO merge_request_ticket_relations VALUES('w','actor-mr','tracking','implements','3');")?;
|
||||||
|
|||||||
@@ -1757,3 +1757,5 @@ WHEN (OLD.runtime_id != NEW.runtime_id OR OLD.worker_id != NEW.worker_id OR OLD.
|
|||||||
WHERE workspace_id = OLD.workspace_id AND runtime_id = OLD.runtime_id
|
WHERE workspace_id = OLD.workspace_id AND runtime_id = OLD.runtime_id
|
||||||
AND worker_id = OLD.worker_id)
|
AND worker_id = OLD.worker_id)
|
||||||
BEGIN SELECT RAISE(ABORT, 'Worker has unfinished Ticket work'); END;
|
BEGIN SELECT RAISE(ABORT, 'Worker has unfinished Ticket work'); END;
|
||||||
|
|
||||||
|
-- Subject deletion schema is composed by store.rs.
|
||||||
|
|||||||
@@ -444,6 +444,28 @@ fn job_result_facts(
|
|||||||
|
|
||||||
pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) {
|
pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) {
|
||||||
use crate::store::JobNotificationFacts;
|
use crate::store::JobNotificationFacts;
|
||||||
|
// A recovered/captured result is not delivery authority. Serialize even the
|
||||||
|
// Runtime send with deletion, so no prepared bytes escape after its fence.
|
||||||
|
let subject = subjektiv_deletion::job_subject(&acceptance.job.request);
|
||||||
|
let gate = match subject
|
||||||
|
.map(|subject| subjektiv_journal::lifecycle_lock(api, subject))
|
||||||
|
.transpose()
|
||||||
|
{
|
||||||
|
Ok(gate) => gate,
|
||||||
|
Err(error) => {
|
||||||
|
tracing::warn!(%error, "Subject Job notification gate unavailable");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let _guard = match gate.as_ref().map(|gate| gate.lock()).transpose() {
|
||||||
|
Ok(guard) => guard,
|
||||||
|
Err(_) => return,
|
||||||
|
};
|
||||||
|
if let Some(subject) = subject
|
||||||
|
&& subjektiv_deletion::require_available(api, subject).is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
|
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -849,7 +849,16 @@ async fn notification_routing_fresh_and_upgraded_schema_and_guards_match() {
|
|||||||
let snapshot = |conn: &Connection| -> Vec<(String, String)> {
|
let snapshot = |conn: &Connection| -> Vec<(String, String)> {
|
||||||
conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::<rusqlite::Result<_>>().unwrap()
|
conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::<rusqlite::Result<_>>().unwrap()
|
||||||
};
|
};
|
||||||
let upgraded = snapshot(&conn);
|
assert_eq!(current_schema_version(&conn).unwrap(), 98);
|
||||||
|
drop(conn);
|
||||||
|
// Compare the complete current composition, not upgraded98 against fresh105.
|
||||||
|
let upgraded_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
let upgraded = upgraded_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
assert_eq!(current_schema_version(conn)?, 105);
|
||||||
|
Ok(snapshot(conn))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
let (fresh_directory, fresh) = tracking_store_fixture().await;
|
let (fresh_directory, fresh) = tracking_store_fixture().await;
|
||||||
fresh
|
fresh
|
||||||
.with_conn(|conn| {
|
.with_conn(|conn| {
|
||||||
|
|||||||
@@ -333,6 +333,15 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
|
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() {
|
||||||
|
let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")?
|
||||||
|
.query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
for id in prior {
|
||||||
|
let prior=load_plan(&tx,&id)?.ok_or_else(||StoreError::Store("removal plan disappeared".into()))?;
|
||||||
|
require_current_subject_deletion_plan(&tx,&prior)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
let mut blockers=Vec::new();
|
let mut blockers=Vec::new();
|
||||||
if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); }
|
if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); }
|
||||||
if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? {
|
if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? {
|
||||||
@@ -363,6 +372,7 @@ impl SqliteWorkspaceStore {
|
|||||||
let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?;
|
let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?;
|
||||||
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
||||||
if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));}
|
if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));}
|
||||||
|
require_current_subject_deletion_plan(&tx, &plan)?;
|
||||||
if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);}
|
if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);}
|
||||||
if plan.state==WorkerRemovalPlanState::Blocked {
|
if plan.state==WorkerRemovalPlanState::Blocked {
|
||||||
return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));
|
return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));
|
||||||
@@ -420,6 +430,8 @@ impl SqliteWorkspaceStore {
|
|||||||
WorkerRetentionError::Invalid("Worker id must be a canonical UUIDv7".to_string())
|
WorkerRetentionError::Invalid("Worker id must be a canonical UUIDv7".to_string())
|
||||||
})?;
|
})?;
|
||||||
let removed_at = plan.created_at.clone();
|
let removed_at = plan.created_at.clone();
|
||||||
|
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
||||||
|
.map_err(map_error)?;
|
||||||
let prior_failure_category = plan.failure_category.clone();
|
let prior_failure_category = plan.failure_category.clone();
|
||||||
Ok(PreparedWorkerRemoval {
|
Ok(PreparedWorkerRemoval {
|
||||||
runtime_request: WorkerRetentionExecutionRequest {
|
runtime_request: WorkerRetentionExecutionRequest {
|
||||||
@@ -459,8 +471,8 @@ impl SqliteWorkspaceStore {
|
|||||||
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END,
|
ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN state='succeeded' THEN 1 ELSE 2 END,
|
||||||
created_at DESC LIMIT 1",
|
created_at DESC,operation_id DESC LIMIT 1",
|
||||||
params![workspace_id, worker.runtime_id, worker.worker_id],
|
params![workspace_id, worker.runtime_id, worker.worker_id],
|
||||||
|row| row.get::<_, String>(0),
|
|row| row.get::<_, String>(0),
|
||||||
)
|
)
|
||||||
@@ -474,6 +486,8 @@ impl SqliteWorkspaceStore {
|
|||||||
let Some(plan) = plan else {
|
let Some(plan) = plan else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
||||||
|
.map_err(map_error)?;
|
||||||
let prior_failure_category = plan.failure_category.clone();
|
let prior_failure_category = plan.failure_category.clone();
|
||||||
let worker_id = plan
|
let worker_id = plan
|
||||||
.worker
|
.worker
|
||||||
@@ -659,6 +673,7 @@ impl SqliteWorkspaceStore {
|
|||||||
let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?;
|
let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?;
|
||||||
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
||||||
if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));}
|
if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));}
|
||||||
|
require_current_subject_deletion_plan(&tx, &plan)?;
|
||||||
if plan.state==WorkerRemovalPlanState::Succeeded{
|
if plan.state==WorkerRemovalPlanState::Succeeded{
|
||||||
// Repair projection fences left by the pre-atomic implementation and
|
// Repair projection fences left by the pre-atomic implementation and
|
||||||
// return a replayable change for a broadcast missed after commit.
|
// return a replayable change for a broadcast missed after commit.
|
||||||
@@ -1024,6 +1039,49 @@ fn load_removal_policy(
|
|||||||
let Some(mut policy) = load_policy(c, w)? else {
|
let Some(mut policy) = load_policy(c, w)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
// Snapshot authority outranks normal policy and Journal cleanup policy. Body
|
||||||
|
// and delegated Sessions must never inherit the ephemeral Job exception.
|
||||||
|
if let Some(is_job) = crate::store::subject_deletion_worker_kind(c, w, worker)? {
|
||||||
|
let update = WorkerRetentionPolicyUpdate {
|
||||||
|
policy_id: if is_job {
|
||||||
|
"subject-deletion-job-v1"
|
||||||
|
} else {
|
||||||
|
"subject-deletion-conversation-v1"
|
||||||
|
}
|
||||||
|
.into(),
|
||||||
|
session_disposition: if is_job {
|
||||||
|
SessionDisposition::Purge
|
||||||
|
} else {
|
||||||
|
SessionDisposition::Archive
|
||||||
|
},
|
||||||
|
metadata_disposition: MetadataDisposition::Tombstone,
|
||||||
|
archive_retention: ArchiveRetention::Forever,
|
||||||
|
// Runtime calls diagnostic archiving Retain, not Archive. Preserve
|
||||||
|
// the Workspace's duration, or thirty days when it normally purges.
|
||||||
|
diagnostics_disposition: if is_job {
|
||||||
|
DiagnosticsDisposition::Purge
|
||||||
|
} else {
|
||||||
|
DiagnosticsDisposition::Retain
|
||||||
|
},
|
||||||
|
diagnostics_retention_seconds: if is_job {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(
|
||||||
|
policy
|
||||||
|
.diagnostics_retention_seconds
|
||||||
|
.unwrap_or(30 * 24 * 60 * 60),
|
||||||
|
)
|
||||||
|
},
|
||||||
|
};
|
||||||
|
policy.policy_id = update.policy_id.clone();
|
||||||
|
policy.policy_digest = worker_retention_policy_digest(&update);
|
||||||
|
policy.session_disposition = update.session_disposition;
|
||||||
|
policy.metadata_disposition = update.metadata_disposition;
|
||||||
|
policy.archive_retention = update.archive_retention;
|
||||||
|
policy.diagnostics_disposition = update.diagnostics_disposition;
|
||||||
|
policy.diagnostics_retention_seconds = update.diagnostics_retention_seconds;
|
||||||
|
return Ok(Some(policy));
|
||||||
|
}
|
||||||
if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? {
|
if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? {
|
||||||
let update = WorkerRetentionPolicyUpdate {
|
let update = WorkerRetentionPolicyUpdate {
|
||||||
policy_id: if erased_journal {
|
policy_id: if erased_journal {
|
||||||
@@ -1047,6 +1105,39 @@ fn load_removal_policy(
|
|||||||
}
|
}
|
||||||
Ok(Some(policy))
|
Ok(Some(policy))
|
||||||
}
|
}
|
||||||
|
fn require_current_subject_deletion_plan(
|
||||||
|
c: &Connection,
|
||||||
|
plan: &WorkerRemovalPlan,
|
||||||
|
) -> crate::Result<()> {
|
||||||
|
if plan.state == WorkerRemovalPlanState::Succeeded
|
||||||
|
&& crate::store::subject_deletion_prior_removal_matches(
|
||||||
|
c,
|
||||||
|
&plan.workspace_id,
|
||||||
|
&plan.worker,
|
||||||
|
&plan.operation_id,
|
||||||
|
)?
|
||||||
|
{
|
||||||
|
// An exact successful receipt pinned before Subject deletion owns its
|
||||||
|
// independent original/archive lifecycle. Never retarget its policy.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() {
|
||||||
|
let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false)?
|
||||||
|
.ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?;
|
||||||
|
if plan.session_disposition != current.session_disposition
|
||||||
|
|| plan.metadata_disposition != current.metadata_disposition
|
||||||
|
|| plan.archive_retention != current.archive_retention
|
||||||
|
|| plan.diagnostics_disposition != current.diagnostics_disposition
|
||||||
|
{
|
||||||
|
return Err(stale_error(
|
||||||
|
plan,
|
||||||
|
"Subject deletion retention differs from saved plan; reconcile canonical Runtime removal before retry",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
||||||
c.query_row(
|
c.query_row(
|
||||||
"SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition,
|
"SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition,
|
||||||
@@ -2736,4 +2827,78 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed);
|
assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() {
|
||||||
|
let store = setup();
|
||||||
|
let ordinary = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||||
|
store
|
||||||
|
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
||||||
|
.unwrap();
|
||||||
|
let body = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||||
|
assert_eq!(body.session_disposition, SessionDisposition::Archive);
|
||||||
|
assert_eq!(body.archive_retention, ArchiveRetention::Forever);
|
||||||
|
assert_eq!(body.metadata_disposition, MetadataDisposition::Tombstone);
|
||||||
|
assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain);
|
||||||
|
assert!(body.diagnostics_retention_seconds.is_some());
|
||||||
|
assert_ne!(body.policy_id, ordinary.policy_id);
|
||||||
|
// A saved plan that would discard diagnostics cannot silently stand in
|
||||||
|
// for the Subject conversation archive plan.
|
||||||
|
assert!(matches!(
|
||||||
|
store.begin_worker_removal("w", &ordinary.plan_id, &ordinary.input_fingerprint),
|
||||||
|
Err(WorkerRetentionError::StalePlan { .. })
|
||||||
|
));
|
||||||
|
let ordinary_store = setup();
|
||||||
|
assert_eq!(
|
||||||
|
ordinary_store
|
||||||
|
.plan_worker_removal(&req(), &inv())
|
||||||
|
.unwrap()
|
||||||
|
.policy_id,
|
||||||
|
ordinary.policy_id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_deletion_job_purge_rejects_recovery_of_old_executing_archive() {
|
||||||
|
let store = setup();
|
||||||
|
let old = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||||
|
store
|
||||||
|
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.with_conn(|c| {
|
||||||
|
c.execute(
|
||||||
|
"UPDATE subject_deletion_workers SET is_job=1 WHERE workspace_id='w'",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
store.recover_worker_removal_execution("w", &req().worker),
|
||||||
|
Err(WorkerRetentionError::StalePlan { .. })
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
store.plan_worker_removal(&req(), &inv()),
|
||||||
|
Err(WorkerRetentionError::StalePlan { .. })
|
||||||
|
));
|
||||||
|
let fresh = store
|
||||||
|
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(fresh.session_disposition, SessionDisposition::Purge);
|
||||||
|
assert_eq!(fresh.diagnostics_disposition, DiagnosticsDisposition::Purge);
|
||||||
|
assert_eq!(fresh.metadata_disposition, MetadataDisposition::Tombstone);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.with_conn(|c| load_plan(c, &old.plan_id))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRemovalPlanState::Executing
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -91,10 +91,16 @@ mod schema_convergence {
|
|||||||
assert_eq!(plan.current_schema_version, 103);
|
assert_eq!(plan.current_schema_version, 103);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
plan.migrations,
|
plan.migrations,
|
||||||
vec![WorkspaceSchemaMigrationStep {
|
vec![
|
||||||
version: 104,
|
WorkspaceSchemaMigrationStep {
|
||||||
name: "Converge Passkey results and Job input retention".into(),
|
version: 104,
|
||||||
}]
|
name: "Converge Passkey results and Job input retention".into(),
|
||||||
|
},
|
||||||
|
WorkspaceSchemaMigrationStep {
|
||||||
|
version: 105,
|
||||||
|
name: "Durable Subject deletion snapshots and Job admission fences".into(),
|
||||||
|
},
|
||||||
|
]
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
schema_rows(&conn),
|
schema_rows(&conn),
|
||||||
@@ -109,6 +115,7 @@ mod schema_convergence {
|
|||||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
let mut expected = history.clone();
|
let mut expected = history.clone();
|
||||||
expected.push((104, "Converge Passkey results and Job input retention".into()));
|
expected.push((104, "Converge Passkey results and Job input retention".into()));
|
||||||
|
expected.push((105, "Durable Subject deletion snapshots and Job admission fences".into()));
|
||||||
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
||||||
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
|
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
|
||||||
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
|
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
|
||||||
@@ -138,7 +145,7 @@ mod schema_convergence {
|
|||||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
workspace_schema_migration_history(conn)?.len(),
|
workspace_schema_migration_history(conn)?.len(),
|
||||||
history.len() + 1
|
history.len() + 2
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
@@ -176,7 +183,7 @@ mod schema_convergence {
|
|||||||
upgraded
|
upgraded
|
||||||
.with_conn(|conn| {
|
.with_conn(|conn| {
|
||||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
assert_eq!(current_schema_version(conn)?, 104);
|
assert_eq!(current_schema_version(conn)?, 105);
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ use subjektiv::api::*;
|
|||||||
mod backend_job_profile;
|
mod backend_job_profile;
|
||||||
mod external_operation_audit;
|
mod external_operation_audit;
|
||||||
mod subject_authorization;
|
mod subject_authorization;
|
||||||
|
mod subjektiv_deletion;
|
||||||
mod subjektiv_job;
|
mod subjektiv_job;
|
||||||
mod subjektiv_journal;
|
mod subjektiv_journal;
|
||||||
mod subjektiv_journal_browser;
|
mod subjektiv_journal_browser;
|
||||||
@@ -4577,6 +4578,12 @@ impl WorkspaceApi {
|
|||||||
.run(move || recover_workdir_removals(&workdir_recovery_api))
|
.run(move || recover_workdir_removals(&workdir_recovery_api))
|
||||||
.await?;
|
.await?;
|
||||||
recover_runtime_removals(&api).await;
|
recover_runtime_removals(&api).await;
|
||||||
|
let deletion_api = api.clone();
|
||||||
|
let deletion_runtime = tokio::runtime::Handle::current();
|
||||||
|
api.host_executors
|
||||||
|
.host()?
|
||||||
|
.run(move || deletion_runtime.block_on(subjektiv_deletion::recover(&deletion_api)))
|
||||||
|
.await?;
|
||||||
api.host_executors
|
api.host_executors
|
||||||
.host()?
|
.host()?
|
||||||
.run(move || {
|
.run(move || {
|
||||||
@@ -4604,9 +4611,23 @@ impl WorkspaceApi {
|
|||||||
) -> Result<BackendJobReservation> {
|
) -> Result<BackendJobReservation> {
|
||||||
self.resolve_backend_job_profile(request)?;
|
self.resolve_backend_job_profile(request)?;
|
||||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||||
let reservation =
|
let reservation = {
|
||||||
|
let gate = subjektiv_deletion::job_subject(request)
|
||||||
|
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||||
|
.transpose()?;
|
||||||
|
let _guard = gate
|
||||||
|
.as_ref()
|
||||||
|
.map(|gate| {
|
||||||
|
gate.lock()
|
||||||
|
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
if let Some(subject) = subjektiv_deletion::job_subject(request) {
|
||||||
|
subjektiv_deletion::require_available(self, subject)?;
|
||||||
|
}
|
||||||
self.store
|
self.store
|
||||||
.reserve_backend_job(&self.config.workspace_id, request, &now)?;
|
.reserve_backend_job(&self.config.workspace_id, request, &now)?
|
||||||
|
};
|
||||||
let result = self.dispatch_backend_job_reservation(reservation);
|
let result = self.dispatch_backend_job_reservation(reservation);
|
||||||
if result.is_err() {
|
if result.is_err() {
|
||||||
self.drain_reserved_backend_jobs();
|
self.drain_reserved_backend_jobs();
|
||||||
@@ -4621,9 +4642,27 @@ impl WorkspaceApi {
|
|||||||
/// immutable request.
|
/// immutable request.
|
||||||
pub fn retry_backend_job(&self, job_id: &str) -> Result<BackendJobReservation> {
|
pub fn retry_backend_job(&self, job_id: &str) -> Result<BackendJobReservation> {
|
||||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||||
let reservation =
|
let reservation = {
|
||||||
|
let current = self
|
||||||
|
.store
|
||||||
|
.get_backend_job(self.workspace_id(), job_id)?
|
||||||
|
.ok_or_else(|| Error::InvalidInput("unknown Backend Job".into()))?;
|
||||||
|
let gate = subjektiv_deletion::job_subject(¤t.request)
|
||||||
|
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||||
|
.transpose()?;
|
||||||
|
let _guard = gate
|
||||||
|
.as_ref()
|
||||||
|
.map(|gate| {
|
||||||
|
gate.lock()
|
||||||
|
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
if let Some(subject) = subjektiv_deletion::job_subject(¤t.request) {
|
||||||
|
subjektiv_deletion::require_available(self, subject)?;
|
||||||
|
}
|
||||||
self.store
|
self.store
|
||||||
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?;
|
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?
|
||||||
|
};
|
||||||
let result = self.dispatch_backend_job_reservation(reservation);
|
let result = self.dispatch_backend_job_reservation(reservation);
|
||||||
if result.is_err() {
|
if result.is_err() {
|
||||||
self.drain_reserved_backend_jobs();
|
self.drain_reserved_backend_jobs();
|
||||||
@@ -4637,13 +4676,8 @@ impl WorkspaceApi {
|
|||||||
) -> Result<BackendJobReservation> {
|
) -> Result<BackendJobReservation> {
|
||||||
// Reread even a terminal/stale reservation before deciding whether it can
|
// Reread even a terminal/stale reservation before deciding whether it can
|
||||||
// dispatch or replay: an explicit erasure may have retired its saved input.
|
// dispatch or replay: an explicit erasure may have retired its saved input.
|
||||||
let journal_gate = reservation
|
let journal_gate = subjektiv_deletion::job_subject(&reservation.job.request)
|
||||||
.job
|
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||||
.request
|
|
||||||
.grants
|
|
||||||
.subjektiv_journal
|
|
||||||
.as_ref()
|
|
||||||
.map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id))
|
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
let _journal_guard = journal_gate
|
let _journal_guard = journal_gate
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -4652,6 +4686,9 @@ impl WorkspaceApi {
|
|||||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
|
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
|
||||||
})
|
})
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
|
if let Some(subject) = subjektiv_deletion::job_subject(&reservation.job.request) {
|
||||||
|
subjektiv_deletion::require_available(self, subject)?;
|
||||||
|
}
|
||||||
let current_job = self
|
let current_job = self
|
||||||
.store
|
.store
|
||||||
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
|
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
|
||||||
@@ -5571,6 +5608,27 @@ impl WorkspaceApi {
|
|||||||
source_worker: &RuntimeWorkerRef,
|
source_worker: &RuntimeWorkerRef,
|
||||||
submission: &BackendJobResultSubmission,
|
submission: &BackendJobResultSubmission,
|
||||||
) -> Result<BackendJobResultAcceptance> {
|
) -> Result<BackendJobResultAcceptance> {
|
||||||
|
let result_job = self
|
||||||
|
.store
|
||||||
|
.get_backend_job(self.workspace_id(), &submission.job_id)?;
|
||||||
|
let subject_gate = result_job
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
|
||||||
|
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||||
|
.transpose()?;
|
||||||
|
let subject_guard = subject_gate
|
||||||
|
.as_ref()
|
||||||
|
.map(|gate| {
|
||||||
|
gate.lock()
|
||||||
|
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
if let Some(subject) = result_job
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
|
||||||
|
{
|
||||||
|
subjektiv_deletion::require_available(self, subject)?;
|
||||||
|
}
|
||||||
let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||||
let mut verified_submission = submission.clone();
|
let mut verified_submission = submission.clone();
|
||||||
if let Some(job) = self
|
if let Some(job) = self
|
||||||
@@ -5696,6 +5754,7 @@ impl WorkspaceApi {
|
|||||||
&verified_submission,
|
&verified_submission,
|
||||||
&accepted_at,
|
&accepted_at,
|
||||||
)?;
|
)?;
|
||||||
|
drop(subject_guard);
|
||||||
if let Some(job) = self
|
if let Some(job) = self
|
||||||
.store
|
.store
|
||||||
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
|
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
|
||||||
@@ -5899,15 +5958,21 @@ impl WorkspaceApi {
|
|||||||
.map(|assignment| assignment.operation_id.clone())
|
.map(|assignment| assignment.operation_id.clone())
|
||||||
})
|
})
|
||||||
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
|
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
|
||||||
let subject = singleton_key
|
let body_subject = singleton_key
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
|
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
|
||||||
if let Some(subject) = subject
|
if let Some(subject) = body_subject
|
||||||
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
|
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
|
||||||
{
|
{
|
||||||
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
|
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
|
||||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||||
}
|
}
|
||||||
|
let subject = body_subject.or_else(|| {
|
||||||
|
request
|
||||||
|
.resolved_execution_grant
|
||||||
|
.as_ref()
|
||||||
|
.map(|grant| grant.subject_id.as_str())
|
||||||
|
});
|
||||||
let subject_gate = subject
|
let subject_gate = subject
|
||||||
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
|
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
@@ -5919,6 +5984,9 @@ impl WorkspaceApi {
|
|||||||
})
|
})
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
if let Some(subject) = subject {
|
if let Some(subject) = subject {
|
||||||
|
subjektiv_deletion::require_available(self, subject)?;
|
||||||
|
}
|
||||||
|
if let Some(subject) = body_subject {
|
||||||
subjektiv_journal::check_start_allowed(self, subject)?;
|
subjektiv_journal::check_start_allowed(self, subject)?;
|
||||||
}
|
}
|
||||||
let reservation = match self.config_store.reserve_worker_create(
|
let reservation = match self.config_store.reserve_worker_create(
|
||||||
@@ -5943,11 +6011,11 @@ impl WorkspaceApi {
|
|||||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
drop(subject_guard);
|
|
||||||
let worker_id = reservation.worker_id;
|
let worker_id = reservation.worker_id;
|
||||||
if let Some(grant) = request.resolved_execution_grant.as_ref() {
|
if let Some(grant) = request.resolved_execution_grant.as_ref() {
|
||||||
subject_authorization::bind_reserved_grant(self, grant, &worker_id.to_string())?;
|
subject_authorization::bind_reserved_grant(self, grant, &worker_id.to_string())?;
|
||||||
}
|
}
|
||||||
|
drop(subject_guard);
|
||||||
// A completed exact-source allocation is historical acceptance, not a new
|
// A completed exact-source allocation is historical acceptance, not a new
|
||||||
// launch. Source/grant authorization and the full fingerprint were checked
|
// launch. Source/grant authorization and the full fingerprint were checked
|
||||||
// above. Do not reserve its already-linked attachments again, redispatch
|
// above. Do not reserve its already-linked attachments again, redispatch
|
||||||
@@ -7458,7 +7526,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou
|
|||||||
.merge(server_api::server_api_axum::subjektiv_memory_changes(
|
.merge(server_api::server_api_axum::subjektiv_memory_changes(
|
||||||
service.clone(),
|
service.clone(),
|
||||||
))
|
))
|
||||||
.merge(server_api::server_api_axum::subjektiv_subject_retire(
|
.merge(server_api::server_api_axum::subjektiv_subject_delete(
|
||||||
service.clone(),
|
service.clone(),
|
||||||
))
|
))
|
||||||
.merge(server_api::server_api_axum::subjektiv_stage_candidate(
|
.merge(server_api::server_api_axum::subjektiv_stage_candidate(
|
||||||
@@ -9724,22 +9792,25 @@ impl server_api::ServerApi for ServerApiContractService {
|
|||||||
.map_err(ApiError::into_repository_api_error)
|
.map_err(ApiError::into_repository_api_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn subjektiv_subject_retire(
|
async fn subjektiv_subject_delete(
|
||||||
&self,
|
&self,
|
||||||
|
context: server_api::ServerRequestContext,
|
||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
subject_id: String,
|
subject_id: String,
|
||||||
) -> std::result::Result<server_api::SubjektivSubjectResponse, server_api::RepositoryApiError>
|
request: server_api::SubjektivSubjectDeleteRequest,
|
||||||
{
|
) -> std::result::Result<
|
||||||
scoped_retire_subjektiv_subject(
|
server_api::SubjektivSubjectDeleteResponse,
|
||||||
State(self.workspace_api()?.clone()),
|
server_api::RepositoryApiError,
|
||||||
AxumPath(ScopedSubjektivSubjectPath {
|
> {
|
||||||
workspace_id,
|
let api = self.workspace_api()?.clone();
|
||||||
subject_id,
|
let actor = subject_authorization::administration_actor(&context)
|
||||||
}),
|
.map_err(ApiError::into_repository_api_error)?;
|
||||||
)
|
require_workspace_owner(&api, &workspace_id, actor, "Subject deletion")
|
||||||
.await
|
.await
|
||||||
.map(|Json(response)| response)
|
.map_err(ApiError::into_repository_api_error)?;
|
||||||
.map_err(ApiError::into_repository_api_error)
|
subjektiv_deletion::delete(api, workspace_id, subject_id, request)
|
||||||
|
.await
|
||||||
|
.map_err(ApiError::into_repository_api_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn subjektiv_stage_candidate(
|
async fn subjektiv_stage_candidate(
|
||||||
@@ -22674,7 +22745,9 @@ fn subjektiv_subject_response(
|
|||||||
memory_fingerprint,
|
memory_fingerprint,
|
||||||
state: match subject.state {
|
state: match subject.state {
|
||||||
crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active,
|
crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active,
|
||||||
crate::subjektiv::SubjectState::Retired => server_api::SubjektivSubjectState::Retired,
|
crate::subjektiv::SubjectState::LegacyRetired => {
|
||||||
|
server_api::SubjektivSubjectState::LegacyRetired
|
||||||
|
}
|
||||||
},
|
},
|
||||||
created_at: subject.created_at,
|
created_at: subject.created_at,
|
||||||
updated_at: subject.updated_at,
|
updated_at: subject.updated_at,
|
||||||
@@ -22962,24 +23035,6 @@ async fn scoped_list_subjektiv_memory_changes(
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn scoped_retire_subjektiv_subject(
|
|
||||||
State(api): State<WorkspaceApi>,
|
|
||||||
AxumPath(path): AxumPath<ScopedSubjektivSubjectPath>,
|
|
||||||
) -> ApiResult<Json<server_api::SubjektivSubjectResponse>> {
|
|
||||||
api.host_executors
|
|
||||||
.host()?
|
|
||||||
.run(move || {
|
|
||||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
|
||||||
let subject = open_subjektiv_store(&api)?
|
|
||||||
.retire_subject(&path.subject_id)
|
|
||||||
.map_err(subjektiv_store_error)?;
|
|
||||||
// Retirement deliberately does not stop or remove the current Worker. The
|
|
||||||
// Worker lease and subject lifecycle are independent authorities.
|
|
||||||
Ok(Json(subjektiv_subject_response(&api, subject)?))
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn scoped_stage_subjektiv_candidate(
|
async fn scoped_stage_subjektiv_candidate(
|
||||||
State(api): State<WorkspaceApi>,
|
State(api): State<WorkspaceApi>,
|
||||||
AxumPath(path): AxumPath<ScopedWorkspacePath>,
|
AxumPath(path): AxumPath<ScopedWorkspacePath>,
|
||||||
@@ -24564,6 +24619,7 @@ fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error {
|
|||||||
Error::WorkspacePermissionDenied(error.to_string())
|
Error::WorkspacePermissionDenied(error.to_string())
|
||||||
}
|
}
|
||||||
crate::subjektiv::SubjektivError::SubjectPermissionsConflict(_)
|
crate::subjektiv::SubjektivError::SubjectPermissionsConflict(_)
|
||||||
|
| crate::subjektiv::SubjektivError::SubjectDeletionConflict(_)
|
||||||
| crate::subjektiv::SubjektivError::PermissionOperationConflict(_)
|
| crate::subjektiv::SubjektivError::PermissionOperationConflict(_)
|
||||||
| crate::subjektiv::SubjektivError::WorkerExecutionGrantConflict(_) => {
|
| crate::subjektiv::SubjektivError::WorkerExecutionGrantConflict(_) => {
|
||||||
Error::RepositoryConflict(error.to_string())
|
Error::RepositoryConflict(error.to_string())
|
||||||
@@ -31543,6 +31599,7 @@ async fn workspace_response(
|
|||||||
manage_secrets: is_owner,
|
manage_secrets: is_owner,
|
||||||
manage_runtimes: is_owner,
|
manage_runtimes: is_owner,
|
||||||
delete_workspace: is_owner,
|
delete_workspace: is_owner,
|
||||||
|
delete_subjects: is_owner,
|
||||||
},
|
},
|
||||||
extension_points: WorkspaceExtensionPoints {
|
extension_points: WorkspaceExtensionPoints {
|
||||||
store: "sqlite".to_string(),
|
store: "sqlite".to_string(),
|
||||||
@@ -37777,6 +37834,7 @@ mod tests {
|
|||||||
mod restore_workdir_tests;
|
mod restore_workdir_tests;
|
||||||
mod restored_parent_control_tests;
|
mod restored_parent_control_tests;
|
||||||
mod subject_authorization_tests;
|
mod subject_authorization_tests;
|
||||||
|
mod subject_deletion_tests;
|
||||||
mod subject_spawn_tests;
|
mod subject_spawn_tests;
|
||||||
mod subjektiv_jobs_tests;
|
mod subjektiv_jobs_tests;
|
||||||
mod subjektiv_journal_tests;
|
mod subjektiv_journal_tests;
|
||||||
@@ -46241,7 +46299,13 @@ mod tests {
|
|||||||
let retired = store
|
let retired = store
|
||||||
.create_subject(crate::subjektiv::SubjectRole::new("retired").unwrap())
|
.create_subject(crate::subjektiv::SubjectRole::new("retired").unwrap())
|
||||||
.unwrap();
|
.unwrap();
|
||||||
store.retire_subject(&retired.id).unwrap();
|
// Frozen lifecycle fixture: migration preserves old inactive Subjects; there is
|
||||||
|
// intentionally no public operation that creates this historical state anymore.
|
||||||
|
api.feature_storage.open(&api.subjektiv_registration).unwrap()
|
||||||
|
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||||
|
tx.execute("UPDATE subjects SET state='retired', record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&retired.id])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
|
||||||
let request = |subject_id: Option<&str>, profile: &str, singleton_key: Option<&str>| {
|
let request = |subject_id: Option<&str>, profile: &str, singleton_key: Option<&str>| {
|
||||||
CreateWorkspaceWorkerRequest {
|
CreateWorkspaceWorkerRequest {
|
||||||
@@ -48843,6 +48907,7 @@ mod tests {
|
|||||||
>,
|
>,
|
||||||
materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
|
materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
|
||||||
spawn_failure: std::sync::Mutex<Option<String>>,
|
spawn_failure: std::sync::Mutex<Option<String>>,
|
||||||
|
subject_delete_failure: std::sync::Mutex<Option<String>>,
|
||||||
recovery_failure: std::sync::Mutex<Option<String>>,
|
recovery_failure: std::sync::Mutex<Option<String>>,
|
||||||
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
|
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
|
||||||
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
|
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
|
||||||
@@ -48887,6 +48952,7 @@ mod tests {
|
|||||||
std::env::temp_dir().join(unique),
|
std::env::temp_dir().join(unique),
|
||||||
),
|
),
|
||||||
spawn_failure: std::sync::Mutex::new(None),
|
spawn_failure: std::sync::Mutex::new(None),
|
||||||
|
subject_delete_failure: std::sync::Mutex::new(None),
|
||||||
recovery_failure: std::sync::Mutex::new(None),
|
recovery_failure: std::sync::Mutex::new(None),
|
||||||
recovery_requests: std::sync::Mutex::new(Vec::new()),
|
recovery_requests: std::sync::Mutex::new(Vec::new()),
|
||||||
recovery_hook: std::sync::Mutex::new(None),
|
recovery_hook: std::sync::Mutex::new(None),
|
||||||
@@ -49242,6 +49308,24 @@ mod tests {
|
|||||||
.map(|execution| execution.snapshot())
|
.map(|execution| execution.snapshot())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn preflight_subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subject_delete_worker(
|
||||||
|
&self,
|
||||||
|
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
if let Some(message) = self.subject_delete_failure.lock().unwrap().clone() {
|
||||||
|
return Err(message);
|
||||||
|
}
|
||||||
|
self.stop_worker(&worker_runtime::identity::WorkerRef::new(request.worker_id));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn stop_worker(
|
fn stop_worker(
|
||||||
&self,
|
&self,
|
||||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
|||||||
@@ -149,6 +149,13 @@ pub(super) fn worker_permissions(
|
|||||||
api: &WorkspaceApi,
|
api: &WorkspaceApi,
|
||||||
worker: &RuntimeWorkerRef,
|
worker: &RuntimeWorkerRef,
|
||||||
) -> ApiResult<Option<Vec<ScopedPermission>>> {
|
) -> ApiResult<Option<Vec<ScopedPermission>>> {
|
||||||
|
if subjektiv_deletion::deleting_worker(api, worker)?
|
||||||
|
|| open_subjektiv_store(api)?
|
||||||
|
.worker_subject_deleted(&worker.runtime_id, &worker.worker_id)
|
||||||
|
.map_err(subjektiv_store_error)?
|
||||||
|
{
|
||||||
|
return Err(denied("Worker belonged to an explicitly deleted Subject"));
|
||||||
|
}
|
||||||
if let Some(grant) = live_child_grant(api, worker)? {
|
if let Some(grant) = live_child_grant(api, worker)? {
|
||||||
return Ok(Some(grant.permissions));
|
return Ok(Some(grant.permissions));
|
||||||
}
|
}
|
||||||
@@ -226,6 +233,8 @@ pub(super) fn prepare_delegation(
|
|||||||
singleton_key: Option<&str>,
|
singleton_key: Option<&str>,
|
||||||
subject_connection: bool,
|
subject_connection: bool,
|
||||||
) -> ApiResult<Option<WorkerExecutionGrant>> {
|
) -> ApiResult<Option<WorkerExecutionGrant>> {
|
||||||
|
// A deleted grant must never become an ordinary non-Subject delegation source.
|
||||||
|
worker_permissions(api, source)?;
|
||||||
let child = live_child_grant(api, source)?;
|
let child = live_child_grant(api, source)?;
|
||||||
let body = if child.is_none() {
|
let body = if child.is_none() {
|
||||||
current_body_subject(api, source)?
|
current_body_subject(api, source)?
|
||||||
|
|||||||
@@ -0,0 +1,288 @@
|
|||||||
|
//! Explicit Subject destruction is independent of ordinary Worker retention. The domain
|
||||||
|
//! fence commits before external effects, remains after failures, and rejects cached grants.
|
||||||
|
//! Worker originals are archived; Job execution copies are purged. Existing archives, Workdirs,
|
||||||
|
//! Tickets and other Subjects are not erased. Archive contents can include previously injected
|
||||||
|
//! Memory: their independent retention/access policy still applies. This is logical content
|
||||||
|
//! deletion, not a promise to erase backups, external providers or already exported material.
|
||||||
|
//!
|
||||||
|
//! Child removal/create-fence receipts, not acceptance, Idle or missing projections, prove
|
||||||
|
//! quiescence. A retry carries the same scope/Subject/operation; no new child identity is issued.
|
||||||
|
//! The domain receipt is authoritative completion. Backend inventories retain only identifiers
|
||||||
|
//! needed for cleanup and fail-closed authorization, never a copy of Subject content.
|
||||||
|
use super::*;
|
||||||
|
use rusqlite::params;
|
||||||
|
|
||||||
|
pub(super) fn job_subject(request: &BackendJobRequest) -> Option<&str> {
|
||||||
|
request
|
||||||
|
.grants
|
||||||
|
.subjektiv_journal
|
||||||
|
.as_ref()
|
||||||
|
.map(|g| g.subject_id.as_str())
|
||||||
|
.or_else(|| {
|
||||||
|
request
|
||||||
|
.grants
|
||||||
|
.subjektiv_consolidation
|
||||||
|
.as_ref()
|
||||||
|
.map(|g| g.subject_id.as_str())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn require_available(api: &WorkspaceApi, subject: &str) -> Result<()> {
|
||||||
|
let store = open_subjektiv_store(api).map_err(|error| error.error)?;
|
||||||
|
if store
|
||||||
|
.subject(subject)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
return Err(Error::SubjektivSubjectNotFound(subject.into()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn pending(api: &WorkspaceApi, subject: &str) -> Result<bool> {
|
||||||
|
api.config_store.with_conn(|connection| {
|
||||||
|
Ok(connection.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2)",
|
||||||
|
params![api.workspace_id(), subject], |row| row.get(0),
|
||||||
|
)?)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This is internal cleanup evidence, never a user-supplied removal flag. It exists only
|
||||||
|
/// after the independently durable domain fence and exact-target inventory commit.
|
||||||
|
pub(super) fn deleting_worker(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<bool> {
|
||||||
|
api.config_store.with_conn(|connection| {
|
||||||
|
Ok(connection.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM subject_deletion_workers WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)",
|
||||||
|
params![api.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get(0),
|
||||||
|
)?)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn delete(
|
||||||
|
api: WorkspaceApi,
|
||||||
|
workspace: String,
|
||||||
|
subject: String,
|
||||||
|
request: server_api::SubjektivSubjectDeleteRequest,
|
||||||
|
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
|
||||||
|
let handle = tokio::runtime::Handle::current();
|
||||||
|
api.host_executors
|
||||||
|
.host()?
|
||||||
|
.run(move || {
|
||||||
|
validate_workspace_scope(&api, &workspace)?;
|
||||||
|
handle.block_on(resume(&api, &subject, &request.operation_id))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn recover(api: &WorkspaceApi) -> Result<()> {
|
||||||
|
let pending = open_subjektiv_store(api)
|
||||||
|
.map_err(|error| error.error)?
|
||||||
|
.pending_subject_deletions()
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?;
|
||||||
|
for deletion in pending {
|
||||||
|
if let Err(error) = resume(api, &deletion.subject_id, &deletion.operation_id).await {
|
||||||
|
tracing::warn!(%error.error, subject_id=%deletion.subject_id,
|
||||||
|
"Subject deletion restart recovery remains incomplete; same request can resume");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resume(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
subject: &str,
|
||||||
|
operation: &str,
|
||||||
|
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
|
||||||
|
let store = open_subjektiv_store(api)?;
|
||||||
|
let gate = subjektiv_journal::lifecycle_lock(api, subject)?;
|
||||||
|
let receipt = {
|
||||||
|
let _guard = gate
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
|
||||||
|
let receipt = store
|
||||||
|
.begin_subject_deletion(subject, operation)
|
||||||
|
.map_err(subjektiv_store_error)?;
|
||||||
|
if !receipt.completed {
|
||||||
|
let delegated = store
|
||||||
|
.deletion_subject_workers(subject)
|
||||||
|
.map_err(subjektiv_store_error)?
|
||||||
|
.into_iter()
|
||||||
|
.map(|(runtime, worker)| RuntimeWorkerRef::new(runtime, worker))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
api.config_store.prepare_subject_deletion(
|
||||||
|
api.workspace_id(),
|
||||||
|
subject,
|
||||||
|
operation,
|
||||||
|
&delegated,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
receipt
|
||||||
|
};
|
||||||
|
let response = |completed| server_api::SubjektivSubjectDeleteResponse {
|
||||||
|
subject_id: subject.into(),
|
||||||
|
operation_id: operation.into(),
|
||||||
|
completed,
|
||||||
|
};
|
||||||
|
if receipt.completed {
|
||||||
|
return Ok(response(true));
|
||||||
|
}
|
||||||
|
// Never hold the Host callback gate across a Runtime call. A create/restore can be
|
||||||
|
// waiting for the callback that now has to reject this permanently fenced Subject.
|
||||||
|
let workers =
|
||||||
|
api.config_store
|
||||||
|
.list_subject_deletion_workers(api.workspace_id(), subject, operation)?;
|
||||||
|
for (worker, _is_job) in workers {
|
||||||
|
if let Err(error) = remove_worker(api, subject, operation, &worker).await {
|
||||||
|
tracing::warn!(%error.error, subject_id=subject, operation_id=operation, "Subject deletion remains incomplete; exact retry required");
|
||||||
|
return Ok(response(false));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let _guard = gate
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
|
||||||
|
api.config_store
|
||||||
|
.erase_subject_deleted_jobs(api.workspace_id(), subject, operation)?;
|
||||||
|
// Finish Backend cleanup before the authoritative domain receipt. If domain
|
||||||
|
// purge fails, its separately committed fence and pending operation survive.
|
||||||
|
// No retry can report domain completion while occupancy/Job cleanup is unproven.
|
||||||
|
api.config_store
|
||||||
|
.finish_subject_deletion_record(api.workspace_id(), subject, operation)?;
|
||||||
|
store
|
||||||
|
.finish_subject_deletion(subject, operation)
|
||||||
|
.map_err(subjektiv_store_error)?;
|
||||||
|
}
|
||||||
|
Ok(response(true))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove_worker(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
subject: &str,
|
||||||
|
operation: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> ApiResult<()> {
|
||||||
|
let request = api.config_store.subject_deletion_worker_request(
|
||||||
|
api.workspace_id(),
|
||||||
|
subject,
|
||||||
|
operation,
|
||||||
|
worker,
|
||||||
|
)?;
|
||||||
|
let fence = api
|
||||||
|
.runtime
|
||||||
|
.subject_delete_worker(&worker.runtime_id, request.clone())
|
||||||
|
.map_err(RuntimeRegistryError::into_error)?;
|
||||||
|
if fence.workspace_id != request.workspace_id
|
||||||
|
|| fence.worker_id != request.worker_id
|
||||||
|
|| fence.create_fingerprint != request.create_fingerprint
|
||||||
|
|| fence.operation_id != request.operation_id
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject Worker deletion fence receipt mismatch".into(),
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
// Replay the canonical child even when physical removal already happened:
|
||||||
|
// its retained receipt can finish the Backend catalog midpoint. The Runtime
|
||||||
|
// fence above also prevents an older create from arriving after that removal.
|
||||||
|
if api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(api.workspace_id(), worker)
|
||||||
|
.map_err(|error| Error::Store(error.to_string()))?
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
return execute_removal(api, operation, worker).await;
|
||||||
|
}
|
||||||
|
match api.runtime.worker(worker) {
|
||||||
|
Ok(detail) => {
|
||||||
|
if !detail.state.eq_ignore_ascii_case("stopped") {
|
||||||
|
let stopped = api
|
||||||
|
.runtime
|
||||||
|
.stop_worker(
|
||||||
|
worker,
|
||||||
|
WorkerLifecycleRequest {
|
||||||
|
reason: Some(format!("Explicit Subject deletion {operation}")),
|
||||||
|
ticket_assignment: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(RuntimeRegistryError::into_error)?;
|
||||||
|
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject Worker stop was not accepted".into(),
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let actual = api
|
||||||
|
.runtime
|
||||||
|
.worker(worker)
|
||||||
|
.map_err(RuntimeRegistryError::into_error)?;
|
||||||
|
if !actual.state.eq_ignore_ascii_case("stopped") {
|
||||||
|
return Err(
|
||||||
|
Error::RepositoryConflict("Subject Worker stop is unconfirmed".into()).into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if api
|
||||||
|
.store
|
||||||
|
.get_worker_registry(api.workspace_id(), worker)?
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
// The Runtime fence authenticated this exact reserved fingerprint. A create
|
||||||
|
// can have reached Runtime before its Backend publication failed. Publish
|
||||||
|
// only its stopped record so the normal archive/Purge owner can remove it.
|
||||||
|
record_worker_summary(
|
||||||
|
api,
|
||||||
|
&actual,
|
||||||
|
&actual.display_name,
|
||||||
|
actual.profile.clone(),
|
||||||
|
WorkerRegistryDisplayNamePolicy::PreserveExisting,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(RuntimeRegistryError::UnknownWorker { .. }) => {
|
||||||
|
if api
|
||||||
|
.store
|
||||||
|
.get_worker_registry(api.workspace_id(), worker)?
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Runtime object missing before canonical original archival".into(),
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
api.config_store.confirm_subject_deletion_absence(
|
||||||
|
api.workspace_id(),
|
||||||
|
subject,
|
||||||
|
operation,
|
||||||
|
worker,
|
||||||
|
&fence,
|
||||||
|
)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error.into_error().into()),
|
||||||
|
}
|
||||||
|
execute_removal(api, operation, worker).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn execute_removal(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
operation: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> ApiResult<()> {
|
||||||
|
let response = WorkerRemovalService::new(api)
|
||||||
|
.execute_target_removal_on_host(
|
||||||
|
api.runtime.as_ref(),
|
||||||
|
worker,
|
||||||
|
&format!("Explicit Subject deletion {operation}"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(Error::Store)?;
|
||||||
|
if response.status != StatusCode::OK.as_u16() {
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject Worker archival/Purge remains unconfirmed".into(),
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -446,6 +446,7 @@ pub(super) fn bind_host_session(
|
|||||||
/// Runs under the same Subject gate as final revoke/singleton release. A failed create or
|
/// Runs under the same Subject gate as final revoke/singleton release. A failed create or
|
||||||
/// legacy removal may have released its Backend reservation but cannot hide a durable writer.
|
/// legacy removal may have released its Backend reservation but cannot hide a durable writer.
|
||||||
pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<()> {
|
pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<()> {
|
||||||
|
subjektiv_deletion::require_available(api, subject)?;
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
let key = crate::subjektiv::subject_worker_singleton_key(subject)
|
let key = crate::subjektiv::subject_worker_singleton_key(subject)
|
||||||
.map_err(|e| Error::InvalidInput(e.to_string()))?;
|
.map_err(|e| Error::InvalidInput(e.to_string()))?;
|
||||||
@@ -1747,6 +1748,13 @@ pub(super) fn prepare_worker_removal(
|
|||||||
runtime: &RuntimeRegistry,
|
runtime: &RuntimeRegistry,
|
||||||
target: &RuntimeWorkerRef,
|
target: &RuntimeWorkerRef,
|
||||||
) -> Result<Option<crate::retention::ErasedJournalSession>> {
|
) -> Result<Option<crate::retention::ErasedJournalSession>> {
|
||||||
|
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||||
|
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||||
|
{
|
||||||
|
// Full Subject deletion has a permanent domain fence. It deliberately drops
|
||||||
|
// Journal continuity, but retains source originals through its archive policy.
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
let Some(attr) = removal_attribution(service, target)? else {
|
let Some(attr) = removal_attribution(service, target)? else {
|
||||||
if service
|
if service
|
||||||
.store
|
.store
|
||||||
@@ -1809,6 +1817,13 @@ pub(super) fn erased_archive_removal_proof(
|
|||||||
target: &RuntimeWorkerRef,
|
target: &RuntimeWorkerRef,
|
||||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
|
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
|
||||||
|
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||||
|
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||||
|
{
|
||||||
|
// Resume the exact canonical retention request after full Subject fencing.
|
||||||
|
// Subject deletion retains originals; it is not per-Session source erasure.
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
let Some(attr) = removal_attribution(service, target)? else {
|
let Some(attr) = removal_attribution(service, target)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
@@ -1852,6 +1867,13 @@ pub(super) fn validate_removal_retention(
|
|||||||
target: &RuntimeWorkerRef,
|
target: &RuntimeWorkerRef,
|
||||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||||
|
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||||
|
{
|
||||||
|
// The plan-local retention policy and canonical Runtime receipt remain required.
|
||||||
|
// Neither unfinished Journals nor a stopped old Controller are deletion blockers.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let Some(attr) = removal_attribution(service, target)? else {
|
let Some(attr) = removal_attribution(service, target)? else {
|
||||||
if service
|
if service
|
||||||
.store
|
.store
|
||||||
@@ -2066,6 +2088,9 @@ pub(super) fn session_erased(api: &WorkspaceApi, request: &BackendJobRequest) ->
|
|||||||
let Some(grant) = &request.grants.subjektiv_journal else {
|
let Some(grant) = &request.grants.subjektiv_journal else {
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
};
|
};
|
||||||
|
if subjektiv_deletion::pending(api, &grant.subject_id)? {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
store
|
store
|
||||||
.journal_session_erased(&host(api, &store, grant)?)
|
.journal_session_erased(&host(api, &store, grant)?)
|
||||||
|
|||||||
@@ -0,0 +1,729 @@
|
|||||||
|
//! Real router/authentication, feature store and Backend DB boundaries. Runtime
|
||||||
|
//! execution/unknown and restart fences are covered in worker-runtime and the
|
||||||
|
//! canonical removal suites, rather than claiming HTTP acceptance proves shutdown.
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
async fn delete_request(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
subject: &str,
|
||||||
|
key: &str,
|
||||||
|
token: Option<&str>,
|
||||||
|
workspace: &str,
|
||||||
|
) -> (StatusCode, Value) {
|
||||||
|
let mut builder = Request::builder()
|
||||||
|
.method("DELETE")
|
||||||
|
.uri(format!("/api/w/{workspace}/subjektiv/subjects/{subject}"))
|
||||||
|
.header(CONTENT_TYPE, "application/json");
|
||||||
|
if let Some(token) = token {
|
||||||
|
builder = builder.header("authorization", format!("Bearer {token}"));
|
||||||
|
}
|
||||||
|
let response = build_router(api.clone())
|
||||||
|
.oneshot(
|
||||||
|
builder
|
||||||
|
.body(Body::from(json!({"operation_id":key}).to_string()))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let bytes = to_bytes(response.into_body(), 2 * 1024 * 1024)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let body = serde_json::from_slice(&bytes)
|
||||||
|
.unwrap_or_else(|_| json!({"text":String::from_utf8_lossy(&bytes)}));
|
||||||
|
(status, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn deletion_owner_scope_exact_retry_and_subject_isolation() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject_with_behavior(
|
||||||
|
crate::subjektiv::SubjectRole::new("delete me").unwrap(),
|
||||||
|
"private behavior".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let other = store
|
||||||
|
.create_subject_with_behavior(
|
||||||
|
crate::subjektiv::SubjectRole::new("keep me").unwrap(),
|
||||||
|
"unchanged".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-owner").await;
|
||||||
|
let stranger = seed_test_api_token(api.store.as_ref(), "subject-delete-stranger");
|
||||||
|
for (token, workspace, expected) in [
|
||||||
|
(None, TEST_WORKSPACE_ID, StatusCode::UNAUTHORIZED),
|
||||||
|
(
|
||||||
|
Some(stranger.as_str()),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
Some(owner.as_str()),
|
||||||
|
"another-workspace",
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let (status, body) =
|
||||||
|
delete_request(&api, &subject.id, "same-operation", token, workspace).await;
|
||||||
|
assert_eq!(status, expected, "{body}");
|
||||||
|
assert!(store.subject(&subject.id).unwrap().is_some());
|
||||||
|
}
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"same-operation",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], true);
|
||||||
|
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||||
|
assert_eq!(
|
||||||
|
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||||
|
"unchanged"
|
||||||
|
);
|
||||||
|
assert_eq!(store.list_subjects(100).unwrap().items.len(), 1);
|
||||||
|
let (status, replay) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"same-operation",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{replay}");
|
||||||
|
assert_eq!(replay, body);
|
||||||
|
let (status, _) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"different-operation",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::CONFLICT);
|
||||||
|
let (status, _) = delete_request(
|
||||||
|
&api,
|
||||||
|
&other.id,
|
||||||
|
"same-operation",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::CONFLICT);
|
||||||
|
assert!(store.subject(&other.id).unwrap().is_some());
|
||||||
|
let fresh = store
|
||||||
|
.create_subject(crate::subjektiv::SubjectRole::new("fresh").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(fresh.id, subject.id);
|
||||||
|
assert!(store.subject_permissions(&fresh.id).unwrap().is_none());
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.pending_staging_candidates(&fresh.id, 100)
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn legacy_inactive_subject_is_preserved_until_explicit_delete_and_retire_route_is_gone() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject_with_behavior(
|
||||||
|
crate::subjektiv::SubjectRole::new("legacy fixture").unwrap(),
|
||||||
|
"retained original".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
api.feature_storage.open(&api.subjektiv_registration).unwrap()
|
||||||
|
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||||
|
tx.execute("UPDATE subjects SET state='retired',record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&subject.id])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store.subject(&subject.id).unwrap().unwrap().state,
|
||||||
|
crate::subjektiv::SubjectState::LegacyRetired
|
||||||
|
);
|
||||||
|
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-legacy").await;
|
||||||
|
let request = Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri(format!(
|
||||||
|
"/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/retire",
|
||||||
|
subject.id
|
||||||
|
))
|
||||||
|
.header("authorization", format!("Bearer {owner}"))
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap();
|
||||||
|
let old = build_router(api.clone()).oneshot(request).await.unwrap();
|
||||||
|
assert_eq!(old.status(), StatusCode::METHOD_NOT_ALLOWED);
|
||||||
|
assert_eq!(
|
||||||
|
store.subject(&subject.id).unwrap().unwrap().behavior_md,
|
||||||
|
"retained original"
|
||||||
|
);
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"explicit-legacy-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], true);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn unconfirmed_worker_shutdown_is_not_completion_and_exact_retry_converges() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
init_clean_git_workspace(dir.path());
|
||||||
|
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject_with_behavior(
|
||||||
|
crate::subjektiv::SubjectRole::new("delete connected body").unwrap(),
|
||||||
|
"private behavior".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let Json(created) = create_workspace_worker(
|
||||||
|
State(api.clone()),
|
||||||
|
HeaderMap::new(),
|
||||||
|
Json(CreateWorkspaceWorkerRequest {
|
||||||
|
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(),
|
||||||
|
display_name: "Subject deletion boundary".into(),
|
||||||
|
singleton_key: None,
|
||||||
|
profile: Some("builtin:companion".into()),
|
||||||
|
ticket_assignment: None,
|
||||||
|
initial_submit: vec![],
|
||||||
|
workdir_attachments: vec![],
|
||||||
|
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
|
||||||
|
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest {
|
||||||
|
subject_id: subject.id.clone(),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
control_operation_id: None,
|
||||||
|
execution_grant: None,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id);
|
||||||
|
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||||
|
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unknown").await;
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = Some("shutdown completion unknown".into());
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"unknown-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], false);
|
||||||
|
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.subject_deletion(&subject.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||||
|
api.config_store.with_conn(|conn| {
|
||||||
|
conn.execute_batch("CREATE TRIGGER injected_worker_catalog_failure BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected catalog failure'); END;")?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
let (status, midpoint) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"unknown-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{midpoint}");
|
||||||
|
assert_eq!(midpoint["completed"], false);
|
||||||
|
assert!(matches!(
|
||||||
|
api.runtime.worker(&worker),
|
||||||
|
Err(RuntimeRegistryError::UnknownWorker { .. })
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.get_worker_registry(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
api.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("DROP TRIGGER injected_worker_catalog_failure;")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"unknown-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], true, "{body}");
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let removal = api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
removal.plan.state,
|
||||||
|
crate::retention::WorkerRemovalPlanState::Succeeded
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
removal.runtime_request.session_disposition,
|
||||||
|
worker_runtime::retention::SessionDisposition::Archive
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.subject_deletion(&subject.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn captured_subject_job_result_cannot_deliver_after_domain_fence() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject(crate::subjektiv::SubjectRole::new("notification fence").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let target = spawn_ticket_check_source(&api, "subject-notification-target");
|
||||||
|
let worker = spawn_ticket_check_source(&api, "subject-notification-result");
|
||||||
|
let request = BackendJobRequest {
|
||||||
|
job_id: "captured-subject-notification".into(),
|
||||||
|
purpose: "subject_notification_contract".into(),
|
||||||
|
input_ref: "subject:notification".into(),
|
||||||
|
input: json!({"private":"input"}),
|
||||||
|
instruction: "Read input".into(),
|
||||||
|
profile: "builtin:backend-job".into(),
|
||||||
|
grants: crate::backend_job::BackendJobGrants {
|
||||||
|
subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant {
|
||||||
|
subject_id: subject.id.clone(),
|
||||||
|
candidate_ids: vec!["candidate".into()],
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
serialization_key: None,
|
||||||
|
source_worker: Some(target.clone()),
|
||||||
|
notification_target: Some(target),
|
||||||
|
limits: Default::default(),
|
||||||
|
};
|
||||||
|
// Enter at the committed result boundary: execution/result validation is
|
||||||
|
// outside this notification test, but routing and Runtime transport are real.
|
||||||
|
let now = Utc::now().to_rfc3339();
|
||||||
|
let reserved = api
|
||||||
|
.store
|
||||||
|
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &now)
|
||||||
|
.unwrap();
|
||||||
|
let attempt = &reserved.attempt.attempt_id;
|
||||||
|
api.store
|
||||||
|
.claim_backend_job_attempt_dispatch(TEST_WORKSPACE_ID, &request.job_id, attempt, &now)
|
||||||
|
.unwrap();
|
||||||
|
api.store
|
||||||
|
.bind_backend_job_attempt_worker(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&request.job_id,
|
||||||
|
attempt,
|
||||||
|
&worker,
|
||||||
|
Some("run"),
|
||||||
|
&now,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let captured = api
|
||||||
|
.store
|
||||||
|
.accept_backend_job_result(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&worker,
|
||||||
|
&BackendJobResultSubmission {
|
||||||
|
job_id: request.job_id.clone(),
|
||||||
|
attempt_id: attempt.clone(),
|
||||||
|
input_digest: request.input_digest().unwrap(),
|
||||||
|
result: json!({"private":"result"}),
|
||||||
|
},
|
||||||
|
&now,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let delivery = crate::backend_job::delivery_id(&request.job_id, attempt);
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.job_notification_source(TEST_WORKSPACE_ID, &request.job_id, attempt)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
execution.take_inputs();
|
||||||
|
store
|
||||||
|
.begin_subject_deletion(&subject.id, "notification-delete")
|
||||||
|
.unwrap();
|
||||||
|
notification_delivery::deliver_job_result(&api, &captured);
|
||||||
|
notification_delivery::recover_job_notifications(&api);
|
||||||
|
assert!(
|
||||||
|
execution.take_inputs().is_empty(),
|
||||||
|
"captured or recovered result must not escape the domain fence"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.job_notification_request(TEST_WORKSPACE_ID, &delivery)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn domain_purge_failure_keeps_fence_and_backend_reopen_resumes_exact_operation() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject_with_behavior(
|
||||||
|
crate::subjektiv::SubjectRole::new("retry purge").unwrap(),
|
||||||
|
"do not resurrect".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let other = store
|
||||||
|
.create_subject_with_behavior(
|
||||||
|
crate::subjektiv::SubjectRole::new("untouched").unwrap(),
|
||||||
|
"keep".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let database = api
|
||||||
|
.feature_storage
|
||||||
|
.open(&api.subjektiv_registration)
|
||||||
|
.unwrap();
|
||||||
|
database.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||||
|
tx.execute_batch("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected domain purge failure'); END;")?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
let owner =
|
||||||
|
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-purge-failure").await;
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"purge-retry",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(status.is_server_error(), "{status}: {body}");
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.subject_deletion(&subject.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||||
|
assert_eq!(
|
||||||
|
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||||
|
"keep"
|
||||||
|
);
|
||||||
|
database
|
||||||
|
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||||
|
tx.execute_batch("DROP TRIGGER injected_subject_purge_failure;")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let mut reopened = api.clone();
|
||||||
|
let config_store =
|
||||||
|
Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap());
|
||||||
|
reopened.config_store = config_store.clone();
|
||||||
|
reopened.store = config_store;
|
||||||
|
let recovery_api = reopened.clone();
|
||||||
|
let handle = tokio::runtime::Handle::current();
|
||||||
|
reopened
|
||||||
|
.host_executors
|
||||||
|
.host()
|
||||||
|
.unwrap()
|
||||||
|
.run(move || handle.block_on(subjektiv_deletion::recover(&recovery_api)))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let recovered = open_subjektiv_store(&reopened)
|
||||||
|
.unwrap()
|
||||||
|
.subject_deletion(&subject.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(recovered.operation_id, "purge-retry");
|
||||||
|
assert!(recovered.completed);
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&reopened,
|
||||||
|
&subject.id,
|
||||||
|
"purge-retry",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], true);
|
||||||
|
assert_eq!(
|
||||||
|
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||||
|
"keep"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn pending_consolidation_is_purged_and_captured_dispatch_and_late_result_are_rejected() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject(crate::subjektiv::SubjectRole::new("delete running Job").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let candidate: memory::extract::StagingRecord = serde_json::from_value(json!({
|
||||||
|
"schema_version": 2, "id": "delete-job-candidate", "extract_run_id": "extract",
|
||||||
|
"source": {"segment_id":"segment", "range":[1,2]}, "kind":"constraint",
|
||||||
|
"claim":"private candidate marker", "why_useful":"retained only until explicit erasure",
|
||||||
|
"evidence": [], "source_refs": []
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.stage_candidate(crate::subjektiv::SubjectStagingRecord::attach(
|
||||||
|
&subject.id,
|
||||||
|
candidate,
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
let started = start_subjektiv_staging_consolidation(
|
||||||
|
api.clone(),
|
||||||
|
&subject.id,
|
||||||
|
MemoryConsolidateStagingOperation { force: true },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(started.status, "started");
|
||||||
|
let job = api
|
||||||
|
.store
|
||||||
|
.find_active_backend_job_for_resource(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&crate::backend_job::subjektiv_serialization_key(&subject.id),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let attempt = api
|
||||||
|
.store
|
||||||
|
.get_backend_job_attempt(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&job.request.job_id,
|
||||||
|
&crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let worker = attempt.worker.clone().unwrap();
|
||||||
|
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||||
|
let captured = BackendJobReservation {
|
||||||
|
job: job.clone(),
|
||||||
|
attempt: attempt.clone(),
|
||||||
|
replayed: true,
|
||||||
|
resource_reused: false,
|
||||||
|
};
|
||||||
|
let owner =
|
||||||
|
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-consolidation").await;
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = Some("completion unknown".into());
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"delete-job",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], false);
|
||||||
|
assert!(
|
||||||
|
api.dispatch_backend_job_reservation(captured.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
api.accept_backend_job_result(
|
||||||
|
&worker,
|
||||||
|
&BackendJobResultSubmission {
|
||||||
|
job_id: job.request.job_id.clone(),
|
||||||
|
attempt_id: attempt.attempt_id.clone(),
|
||||||
|
input_digest: job.input_digest.clone(),
|
||||||
|
result: json!({"late":true}),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"delete-job",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], true);
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.get_backend_job_attempt(TEST_WORKSPACE_ID, &job.request.job_id, &attempt.attempt_id)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(api.dispatch_backend_job_reservation(captured).is_err());
|
||||||
|
assert!(api.dispatch_backend_job(&job.request).is_err());
|
||||||
|
let removal = api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
removal.plan.state,
|
||||||
|
crate::retention::WorkerRemovalPlanState::Succeeded
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
removal.runtime_request.session_disposition,
|
||||||
|
worker_runtime::retention::SessionDisposition::Purge
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
removal.runtime_request.diagnostics_disposition,
|
||||||
|
worker_runtime::retention::DiagnosticsDisposition::Purge
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!dir.path()
|
||||||
|
.join(".test-embedded-runtime-store/workers")
|
||||||
|
.join(&worker.worker_id)
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unmaterialized_body_and_job_reservations_finish_only_with_runtime_storage_absence_proof() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, _) = test_api_with_recording_backend(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let owner =
|
||||||
|
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unmaterialized").await;
|
||||||
|
for is_job in [false, true] {
|
||||||
|
let subject = store
|
||||||
|
.create_subject(crate::subjektiv::SubjectRole::new("reserved before deletion").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let singleton = (!is_job).then(|| format!("subjektiv:{}", subject.id));
|
||||||
|
let allocation = if is_job {
|
||||||
|
let request = BackendJobRequest {
|
||||||
|
job_id: "unmaterialized-subject-job".into(),
|
||||||
|
purpose: "subject_reservation_contract".into(),
|
||||||
|
input_ref: "subject:reserved".into(),
|
||||||
|
input: json!({"private":"reserved input"}),
|
||||||
|
instruction: "read".into(),
|
||||||
|
profile: "builtin:backend-job".into(),
|
||||||
|
grants: crate::backend_job::BackendJobGrants {
|
||||||
|
subjektiv_consolidation: Some(
|
||||||
|
crate::backend_job::SubjektivConsolidationGrant {
|
||||||
|
subject_id: subject.id.clone(),
|
||||||
|
candidate_ids: vec!["candidate".into()],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
serialization_key: None,
|
||||||
|
source_worker: None,
|
||||||
|
notification_target: None,
|
||||||
|
limits: Default::default(),
|
||||||
|
};
|
||||||
|
let reserved = api
|
||||||
|
.store
|
||||||
|
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339())
|
||||||
|
.unwrap();
|
||||||
|
crate::backend_job::allocation_key(&reserved.attempt.attempt_id)
|
||||||
|
} else {
|
||||||
|
"unmaterialized-subject-body".into()
|
||||||
|
};
|
||||||
|
let settings = api
|
||||||
|
.config_store
|
||||||
|
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
|
||||||
|
.unwrap();
|
||||||
|
let reserved = api
|
||||||
|
.config_store
|
||||||
|
.reserve_worker_create(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
EMBEDDED_WORKER_RUNTIME_ID,
|
||||||
|
&allocation,
|
||||||
|
"unmaterialized-fingerprint",
|
||||||
|
singleton.as_deref(),
|
||||||
|
&settings,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let worker =
|
||||||
|
RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, reserved.worker_id.to_string());
|
||||||
|
assert!(matches!(
|
||||||
|
api.runtime.worker(&worker),
|
||||||
|
Err(RuntimeRegistryError::UnknownWorker { .. })
|
||||||
|
));
|
||||||
|
let (status, body) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
&format!("delete-reserved-{is_job}"),
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(body["completed"], true);
|
||||||
|
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
|
||||||
|
let proved = api.config_store.with_conn(|conn| Ok(conn.query_row("SELECT originals_absent FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", rusqlite::params![TEST_WORKSPACE_ID,subject.id,worker.runtime_id,worker.worker_id], |row|row.get::<_,bool>(0))?)).unwrap();
|
||||||
|
assert!(proved);
|
||||||
|
assert!(
|
||||||
|
api.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.is_none(),
|
||||||
|
"absence is not a fabricated Purge/archive receipt"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
api.config_store
|
||||||
|
.reserve_worker_create(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
EMBEDDED_WORKER_RUNTIME_ID,
|
||||||
|
&allocation,
|
||||||
|
"unmaterialized-fingerprint",
|
||||||
|
singleton.as_deref(),
|
||||||
|
&settings
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
if let Some(singleton) = singleton {
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1143,6 +1143,8 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
|||||||
&Utc::now().to_rfc3339(),
|
&Utc::now().to_rfc3339(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
// Discard only the original delivery, before recovery can automatically retry.
|
||||||
|
f.execution.take_inputs();
|
||||||
f.api.recover_backend_job_worker_cleanups().unwrap();
|
f.api.recover_backend_job_worker_cleanups().unwrap();
|
||||||
wait_for_backend_job_worker_cleanup(
|
wait_for_backend_job_worker_cleanup(
|
||||||
&f.api,
|
&f.api,
|
||||||
@@ -1151,23 +1153,64 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
|||||||
BackendJobWorkerCleanupState::Completed,
|
BackendJobWorkerCleanupState::Completed,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
// Cleanup commits before domain reconciliation; wait for its authoritative outcome.
|
// Cleanup commits before domain reconciliation. Job Failed is only transient:
|
||||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
// reconcile_unknown fences this attempt, then automatically reserves/queues its
|
||||||
|
// retry. Observe the immutable attempt outcome and the delivered new generation,
|
||||||
|
// not the Job's transient state; a manual retry would race the automatic one.
|
||||||
|
let reconciliation = tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
loop {
|
loop {
|
||||||
let job = f
|
let original = f.api.store.get_backend_job_attempt(
|
||||||
.api
|
TEST_WORKSPACE_ID,
|
||||||
.store
|
&f.request.job_id,
|
||||||
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
&dispatched.attempt.attempt_id,
|
||||||
.unwrap()
|
).unwrap().unwrap();
|
||||||
.unwrap();
|
if original.state == BackendJobAttemptState::Failed {
|
||||||
if job.state == BackendJobState::Failed {
|
assert_eq!(original.failure_category.as_deref(), Some("journal_attempt_fenced"));
|
||||||
break;
|
assert_eq!(original.worker_cleanup_state, Some(BackendJobWorkerCleanupState::Completed));
|
||||||
|
assert!(original.worker_cleanup_completed_at.is_some());
|
||||||
|
if let Some(generation) = f.store.active_journal_generation(&f.host, &f.id).unwrap()
|
||||||
|
&& generation.job.attempt_id != original.attempt_id
|
||||||
|
&& let Some(retry) = f.api.store.get_backend_job_attempt(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&f.request.job_id,
|
||||||
|
&generation.job.attempt_id,
|
||||||
|
).unwrap()
|
||||||
|
&& retry.state == BackendJobAttemptState::Dispatched
|
||||||
|
&& let Some(resumed) = &retry.worker
|
||||||
|
// Worker binding precedes input delivery/attestation. ReadState
|
||||||
|
// succeeds only once the new generation's delivery is attested.
|
||||||
|
&& subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()
|
||||||
|
{
|
||||||
|
break (original, retry, generation);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tokio::task::yield_now().await;
|
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.unwrap();
|
assert!(
|
||||||
|
reconciliation.is_ok(),
|
||||||
|
"post-cleanup reconciliation timed out: job={:?}, original_attempt={:?}, active_generation={:?}",
|
||||||
|
f.api
|
||||||
|
.store
|
||||||
|
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||||
|
.unwrap(),
|
||||||
|
f.api
|
||||||
|
.store
|
||||||
|
.get_backend_job_attempt(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&f.request.job_id,
|
||||||
|
&dispatched.attempt.attempt_id
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
f.store.active_journal_generation(&f.host, &f.id).unwrap(),
|
||||||
|
);
|
||||||
|
let (fenced, retry, generation) = reconciliation.unwrap();
|
||||||
|
assert_eq!(fenced.worker, dispatched.attempt.worker);
|
||||||
|
assert_eq!(retry.attempt, dispatched.attempt.attempt + 1);
|
||||||
|
assert_eq!(generation.job.job_id, f.request.job_id);
|
||||||
|
assert_eq!(generation.job.attempt_id, retry.attempt_id);
|
||||||
|
assert_eq!(retry.input_digest, f.request.input_digest().unwrap());
|
||||||
f.assert_purged(worker);
|
f.assert_purged(worker);
|
||||||
assert!(f.store.journal_retention_required(&f.host).unwrap());
|
assert!(f.store.journal_retention_required(&f.host).unwrap());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -1187,25 +1230,25 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
|||||||
1,
|
1,
|
||||||
"source pin survives failed/partial Job cleanup"
|
"source pin survives failed/partial Job cleanup"
|
||||||
);
|
);
|
||||||
f.execution.take_inputs();
|
let job = f
|
||||||
let retry = f
|
|
||||||
.api
|
.api
|
||||||
.store
|
.store
|
||||||
.reserve_backend_job_retry(
|
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||||
TEST_WORKSPACE_ID,
|
.unwrap()
|
||||||
&f.request.job_id,
|
|
||||||
&Utc::now().to_rfc3339(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(retry.job.request.input, f.request.input);
|
assert_eq!(job.state, BackendJobState::Pending);
|
||||||
assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap());
|
assert_eq!(job.current_attempt, retry.attempt);
|
||||||
let retry = f.api.dispatch_backend_job_reservation(retry).unwrap();
|
assert_eq!(
|
||||||
let resumed = retry.attempt.worker.as_ref().unwrap();
|
job.load_executable_request().unwrap().input,
|
||||||
|
f.request.input
|
||||||
|
);
|
||||||
|
assert_eq!(job.input_digest, f.request.input_digest().unwrap());
|
||||||
|
let resumed = retry.worker.as_ref().unwrap();
|
||||||
let delivered = f.execution.take_inputs();
|
let delivered = f.execution.take_inputs();
|
||||||
assert_eq!(delivered.len(), 1);
|
assert_eq!(delivered.len(), 1);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
delivered[0].1,
|
delivered[0].1,
|
||||||
f.request.worker_input(&retry.attempt.attempt_id).unwrap()
|
f.request.worker_input(&retry.attempt_id).unwrap()
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
f.store
|
f.store
|
||||||
|
|||||||
@@ -92,6 +92,13 @@ use crate::backend_job::{
|
|||||||
use crate::workspace_deletion::WorkspaceDeletionStore;
|
use crate::workspace_deletion::WorkspaceDeletionStore;
|
||||||
use crate::{Error, Result};
|
use crate::{Error, Result};
|
||||||
|
|
||||||
|
#[path = "subject_deletion_store.rs"]
|
||||||
|
mod subject_deletion_store;
|
||||||
|
pub(crate) use subject_deletion_store::{
|
||||||
|
prior_removal_matches as subject_deletion_prior_removal_matches,
|
||||||
|
worker_kind as subject_deletion_worker_kind,
|
||||||
|
};
|
||||||
|
|
||||||
#[path = "activation.rs"]
|
#[path = "activation.rs"]
|
||||||
mod activation;
|
mod activation;
|
||||||
pub use activation::ActivationCodeIssue;
|
pub use activation::ActivationCodeIssue;
|
||||||
@@ -119,7 +126,7 @@ pub use ticket::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||||
const LATEST_SCHEMA_VERSION: i64 = 104;
|
const LATEST_SCHEMA_VERSION: i64 = 105;
|
||||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||||
@@ -447,6 +454,11 @@ const MIGRATIONS: &[Migration] = &[
|
|||||||
name: "Converge Passkey results and Job input retention",
|
name: "Converge Passkey results and Job input retention",
|
||||||
apply: migrate_schema_103_variants_to_v104,
|
apply: migrate_schema_103_variants_to_v104,
|
||||||
},
|
},
|
||||||
|
Migration {
|
||||||
|
version: 105,
|
||||||
|
name: "Durable Subject deletion snapshots and Job admission fences",
|
||||||
|
apply: subject_deletion_store::migrate,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
||||||
@@ -3000,6 +3012,7 @@ impl SqliteWorkspaceStore {
|
|||||||
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
subject_deletion_store::require_worker_admission(&tx, workspace_id, allocation_key, singleton_key)?;
|
||||||
let fixed_worker_id = prepare(&tx)?;
|
let fixed_worker_id = prepare(&tx)?;
|
||||||
let existing = tx
|
let existing = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
@@ -7455,6 +7468,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
let deadline = backend_job_deadline(now, request.limits.timeout_seconds)?;
|
let deadline = backend_job_deadline(now, request.limits.timeout_seconds)?;
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, request)?;
|
||||||
if let Some(job) = read_backend_job(&tx, workspace_id, &request.job_id)? {
|
if let Some(job) = read_backend_job(&tx, workspace_id, &request.job_id)? {
|
||||||
if job.intent_fingerprint != fingerprint {
|
if job.intent_fingerprint != fingerprint {
|
||||||
return Err(Error::InvalidInput(format!(
|
return Err(Error::InvalidInput(format!(
|
||||||
@@ -7532,7 +7546,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
return Ok(job);
|
return Ok(job);
|
||||||
}
|
}
|
||||||
if reason == JobInputRetirementReason::Completed {
|
if reason == JobInputRetirementReason::Completed {
|
||||||
job.load_executable_request()?;
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
|
job.load_executable_request()?;
|
||||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
|
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
|
||||||
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
|
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
|
||||||
if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed
|
if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed
|
||||||
@@ -7562,6 +7577,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let mut job = read_backend_job(&tx, workspace_id, job_id)?
|
let mut job = read_backend_job(&tx, workspace_id, job_id)?
|
||||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
let previous_id = attempt_id(job_id, job.current_attempt);
|
let previous_id = attempt_id(job_id, job.current_attempt);
|
||||||
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
|
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
|
||||||
@@ -7701,6 +7717,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||||
@@ -7764,6 +7781,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||||
@@ -7827,6 +7845,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
|
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||||
.ok_or_else(|| {
|
.ok_or_else(|| {
|
||||||
@@ -7880,6 +7899,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{}`", submission.job_id)))?;
|
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{}`", submission.job_id)))?;
|
||||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
|
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
|
||||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
|
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
|
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
|
||||||
if submission.input_digest != job.input_digest
|
if submission.input_digest != job.input_digest
|
||||||
|| attempt.input_digest != submission.input_digest
|
|| attempt.input_digest != submission.input_digest
|
||||||
@@ -7908,6 +7928,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
|
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
|
||||||
}
|
}
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
|
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
|
||||||
return Err(Error::InvalidInput(
|
return Err(Error::InvalidInput(
|
||||||
@@ -7972,6 +7993,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
|
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
|
||||||
|
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
|
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
|
||||||
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
|
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
|
||||||
if &attempt != expected || job.current_attempt != attempt.attempt
|
if &attempt != expected || job.current_attempt != attempt.attempt
|
||||||
@@ -8277,6 +8299,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
) -> Result<(BackendJobDeliveryRecord, bool)> {
|
) -> Result<(BackendJobDeliveryRecord, bool)> {
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let deleted:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id=?1 AND job_id=?2)",params![workspace_id,job_id],|r|r.get(0))?;
|
||||||
|
if deleted {return Err(Error::RepositoryConflict("subject_deleted: Job delivery fenced".into()));}
|
||||||
if let Some(existing) = read_backend_job_delivery(&tx, workspace_id, delivery_id)? {
|
if let Some(existing) = read_backend_job_delivery(&tx, workspace_id, delivery_id)? {
|
||||||
if existing.job_id != job_id || existing.target != *target {
|
if existing.job_id != job_id || existing.target != *target {
|
||||||
return Err(Error::InvalidInput(
|
return Err(Error::InvalidInput(
|
||||||
@@ -15545,6 +15569,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
|||||||
notification_routing::jobs::install(conn)?;
|
notification_routing::jobs::install(conn)?;
|
||||||
notification_routing::install_business_events(conn)?;
|
notification_routing::install_business_events(conn)?;
|
||||||
notification_routing::upgrade_result_reports(conn)?;
|
notification_routing::upgrade_result_reports(conn)?;
|
||||||
|
subject_deletion_store::install_notification_erasure(conn)?;
|
||||||
ticket::verify_sqlite_ticket_schema(conn).map_err(|error| {
|
ticket::verify_sqlite_ticket_schema(conn).map_err(|error| {
|
||||||
Error::Store(format!(
|
Error::Store(format!(
|
||||||
"fresh Workspace Ticket schema verification failed: {error}"
|
"fresh Workspace Ticket schema verification failed: {error}"
|
||||||
@@ -15583,10 +15608,17 @@ fn create_workspace_schema_structure_from_sql(
|
|||||||
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
||||||
|
|
||||||
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
|
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
|
||||||
conn.execute_batch(&schema_sql.replace(
|
conn.execute_batch(
|
||||||
"-- Ticket-owned assignment schema is composed by store.rs.",
|
&schema_sql
|
||||||
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
|
.replace(
|
||||||
))?;
|
"-- Ticket-owned assignment schema is composed by store.rs.",
|
||||||
|
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
|
||||||
|
)
|
||||||
|
.replace(
|
||||||
|
"-- Subject deletion schema is composed by store.rs.",
|
||||||
|
include_str!("subject_deletion_schema.sql"),
|
||||||
|
),
|
||||||
|
)?;
|
||||||
let composed_sql = r#"
|
let composed_sql = r#"
|
||||||
ALTER TABLE typed_tickets RENAME TO typed_tickets_legacy_v66;
|
ALTER TABLE typed_tickets RENAME TO typed_tickets_legacy_v66;
|
||||||
CREATE TABLE typed_tickets (
|
CREATE TABLE typed_tickets (
|
||||||
@@ -15809,6 +15841,9 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> {
|
|||||||
if current_schema_version(conn)? >= 96 {
|
if current_schema_version(conn)? >= 96 {
|
||||||
notification_routing::jobs::verify(conn)?;
|
notification_routing::jobs::verify(conn)?;
|
||||||
}
|
}
|
||||||
|
if current_schema_version(conn)? >= 105 {
|
||||||
|
subject_deletion_store::verify(conn)?;
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -16223,6 +16258,8 @@ fn current_worker_singleton_owner(
|
|||||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
||||||
|
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \
|
||||||
|
AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \
|
||||||
THEN 'created' ELSE reservation.state END \
|
THEN 'created' ELSE reservation.state END \
|
||||||
FROM worker_singleton_owners owner \
|
FROM worker_singleton_owners owner \
|
||||||
LEFT JOIN worker_create_reservations reservation \
|
LEFT JOIN worker_create_reservations reservation \
|
||||||
@@ -19265,6 +19302,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
|
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
|
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
|
||||||
|
WorkspaceSchemaMigrationStep {version:105, name:"Durable Subject deletion snapshots and Job admission fences".into()},
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -19379,6 +19417,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
(102, "Durable failed Subject creation recovery".to_string()),
|
(102, "Durable failed Subject creation recovery".to_string()),
|
||||||
(103, "Passkey metadata and atomic addition results".to_string()),
|
(103, "Passkey metadata and atomic addition results".to_string()),
|
||||||
(104, "Converge Passkey results and Job input retention".to_string()),
|
(104, "Converge Passkey results and Job input retention".to_string()),
|
||||||
|
(105, "Durable Subject deletion snapshots and Job admission fences".to_string()),
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||||
@@ -22664,7 +22703,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
(101, "Durable explicit Subject body release".into()),
|
(101, "Durable explicit Subject body release".into()),
|
||||||
(102, "Durable failed Subject creation recovery".into()),
|
(102, "Durable failed Subject creation recovery".into()),
|
||||||
(103, "Passkey metadata and atomic addition results".into()),
|
(103, "Passkey metadata and atomic addition results".into()),
|
||||||
(104, "Converge Passkey results and Job input retention".into())]);
|
(104, "Converge Passkey results and Job input retention".into()),
|
||||||
|
(105, "Durable Subject deletion snapshots and Job admission fences".into())]);
|
||||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||||
}
|
}
|
||||||
@@ -26857,12 +26897,19 @@ INSERT INTO worker_registry (
|
|||||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
store.with_conn(|conn| {
|
store.with_conn(|conn| {
|
||||||
|
// This fixture deliberately reconstructs schema 102 from fresh.
|
||||||
|
// Remove the newer Subject fence composition, not just its receipt.
|
||||||
|
let triggers=conn.prepare("SELECT name,tbl_name FROM sqlite_schema WHERE type='trigger' AND tbl_name LIKE 'notification_%' AND name NOT LIKE 'subject_deleted_%' AND sql LIKE '%subject_deletion_jobs%'")?
|
||||||
|
.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;}
|
||||||
|
for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;}
|
||||||
|
conn.execute_batch("DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?;
|
||||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
DROP TRIGGER passkey_metadata_insert;
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
DROP TABLE passkey_addition_receipts;
|
DROP TABLE passkey_addition_receipts;
|
||||||
DROP TABLE passkey_metadata;
|
DROP TABLE passkey_metadata;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=105;
|
||||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
drop(store);
|
drop(store);
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
-- Content-free, monotonic main-DB fence. The Host installs the domain fence first
|
||||||
|
-- and holds its Subject gate across snapshot/admission. No Subject contents live here.
|
||||||
|
CREATE TABLE subject_deletions (
|
||||||
|
workspace_id TEXT NOT NULL,
|
||||||
|
subject_id TEXT NOT NULL,
|
||||||
|
operation_id TEXT NOT NULL CHECK(length(operation_id)>0),
|
||||||
|
completed INTEGER NOT NULL DEFAULT 0 CHECK(completed IN (0,1)),
|
||||||
|
jobs_erased INTEGER NOT NULL DEFAULT 0 CHECK(jobs_erased IN (0,1)),
|
||||||
|
PRIMARY KEY(workspace_id,subject_id),
|
||||||
|
UNIQUE(workspace_id,operation_id),
|
||||||
|
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE TABLE subject_deletion_workers (
|
||||||
|
workspace_id TEXT NOT NULL,
|
||||||
|
subject_id TEXT NOT NULL,
|
||||||
|
runtime_id TEXT NOT NULL,
|
||||||
|
worker_id TEXT NOT NULL,
|
||||||
|
is_job INTEGER NOT NULL CHECK(is_job IN (0,1)),
|
||||||
|
-- Exact canonical success already present at prepare, never a later receipt.
|
||||||
|
prior_removal_operation_id TEXT REFERENCES worker_removal_operations(operation_id),
|
||||||
|
-- Confirmed by the scoped Runtime storage owner, not catalog absence or Stop ACK.
|
||||||
|
originals_absent INTEGER NOT NULL DEFAULT 0 CHECK(originals_absent IN (0,1)),
|
||||||
|
PRIMARY KEY(workspace_id,subject_id,runtime_id,worker_id),
|
||||||
|
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE TRIGGER subject_deletion_worker_prior_insert BEFORE INSERT ON subject_deletion_workers
|
||||||
|
WHEN NEW.prior_removal_operation_id IS NOT NULL AND NOT EXISTS(
|
||||||
|
SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=NEW.prior_removal_operation_id
|
||||||
|
AND r.workspace_id=NEW.workspace_id AND r.runtime_id=NEW.runtime_id
|
||||||
|
AND r.worker_id=NEW.worker_id AND r.state='succeeded')
|
||||||
|
BEGIN SELECT RAISE(ABORT,'Subject deletion prior removal must be an exact canonical success'); END;
|
||||||
|
CREATE TRIGGER subject_deletion_worker_identity_immutable BEFORE UPDATE OF
|
||||||
|
workspace_id,subject_id,runtime_id,worker_id,prior_removal_operation_id ON subject_deletion_workers
|
||||||
|
BEGIN SELECT RAISE(ABORT,'Subject deletion Worker snapshot identity is immutable'); END;
|
||||||
|
CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers
|
||||||
|
WHEN OLD.originals_absent=1 AND NEW.originals_absent=0
|
||||||
|
BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END;
|
||||||
|
-- Job/attempt identities remain after payload erasure, preventing stale allocation
|
||||||
|
-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs.
|
||||||
|
CREATE TABLE subject_deletion_jobs (
|
||||||
|
workspace_id TEXT NOT NULL,
|
||||||
|
subject_id TEXT NOT NULL,
|
||||||
|
job_id TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(workspace_id,job_id),
|
||||||
|
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE TABLE subject_deletion_allocations (
|
||||||
|
workspace_id TEXT NOT NULL,
|
||||||
|
subject_id TEXT NOT NULL,
|
||||||
|
allocation_key TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(workspace_id,allocation_key),
|
||||||
|
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE TRIGGER subject_deleted_job_insert BEFORE INSERT ON backend_jobs
|
||||||
|
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
|
||||||
|
(d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_journal.subject_id') OR
|
||||||
|
d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_consolidation.subject_id')))
|
||||||
|
OR EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||||
|
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job admission fenced'); END;
|
||||||
|
CREATE TRIGGER subject_deleted_job_result BEFORE UPDATE OF state,result_json ON backend_jobs
|
||||||
|
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||||
|
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job result fenced'); END;
|
||||||
|
CREATE TRIGGER subject_deleted_attempt_insert BEFORE INSERT ON backend_job_attempts
|
||||||
|
WHEN EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||||
|
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job retry fenced'); END;
|
||||||
|
CREATE TRIGGER subject_deleted_attempt_result BEFORE UPDATE OF state,result_json,runtime_id,worker_id,runtime_run_id ON backend_job_attempts
|
||||||
|
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||||
|
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job attempt fenced'); END;
|
||||||
|
CREATE TRIGGER subject_deleted_worker_reservation BEFORE INSERT ON worker_create_reservations
|
||||||
|
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
|
||||||
|
NEW.singleton_key IN ('subjektiv:'||d.subject_id,'subjektiv-consolidation:'||d.subject_id))
|
||||||
|
OR EXISTS(SELECT 1 FROM subject_deletion_allocations d WHERE d.workspace_id=NEW.workspace_id AND d.allocation_key=NEW.allocation_key)
|
||||||
|
BEGIN SELECT RAISE(ABORT,'subject_deleted: Worker allocation fenced'); END;
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -94,6 +94,12 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
|||||||
"workdir_registry",
|
"workdir_registry",
|
||||||
"workdir_removal_operations",
|
"workdir_removal_operations",
|
||||||
"worker_control_grants",
|
"worker_control_grants",
|
||||||
|
// Snapshot children precede their fence parent and canonical Worker removal
|
||||||
|
// receipts (subject_deletion_workers.prior_removal_operation_id).
|
||||||
|
"subject_deletion_allocations",
|
||||||
|
"subject_deletion_jobs",
|
||||||
|
"subject_deletion_workers",
|
||||||
|
"subject_deletions",
|
||||||
"subject_worker_releases",
|
"subject_worker_releases",
|
||||||
"subject_create_recoveries",
|
"subject_create_recoveries",
|
||||||
"worker_create_reservations",
|
"worker_create_reservations",
|
||||||
@@ -1276,6 +1282,134 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() {
|
||||||
|
let (store, owner, workspace_id) = setup();
|
||||||
|
let tables = [
|
||||||
|
"subject_deletion_allocations",
|
||||||
|
"subject_deletion_jobs",
|
||||||
|
"subject_deletion_workers",
|
||||||
|
"subject_deletions",
|
||||||
|
"worker_removal_operations",
|
||||||
|
];
|
||||||
|
// Check the explicit FK ordering as well as execution (the purge defers FKs).
|
||||||
|
let position = |table| {
|
||||||
|
WORKSPACE_DELETION_PURGE_TABLES
|
||||||
|
.iter()
|
||||||
|
.position(|name| *name == table)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
for child in &tables[..3] {
|
||||||
|
assert!(position(*child) < position("subject_deletions"));
|
||||||
|
}
|
||||||
|
assert!(position("subject_deletion_workers") < position("worker_removal_operations"));
|
||||||
|
for workspace in ["workspace-a", "workspace-b"] {
|
||||||
|
store.with_conn(|conn| {
|
||||||
|
conn.execute("INSERT INTO worker_removal_operations(
|
||||||
|
operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id,
|
||||||
|
worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,
|
||||||
|
archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at)
|
||||||
|
VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest',
|
||||||
|
'purge','purge','forever','purge','[]','succeeded','removed','1','1')", [workspace])?;
|
||||||
|
conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased)
|
||||||
|
VALUES(?1,'subject','subject-delete',1,1)", [workspace])?;
|
||||||
|
conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent)
|
||||||
|
VALUES(?1,'subject','runtime','worker',1,?1,1)", [workspace])?;
|
||||||
|
conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?;
|
||||||
|
conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
|
let rows = |workspace: &str| {
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
tables
|
||||||
|
.iter()
|
||||||
|
.map(|table| {
|
||||||
|
let mut statement = conn
|
||||||
|
.prepare(&format!("SELECT * FROM {table} WHERE workspace_id=?1"))?;
|
||||||
|
let count = statement.column_count();
|
||||||
|
Ok(statement
|
||||||
|
.query_map([workspace], |row| {
|
||||||
|
(0..count)
|
||||||
|
.map(|index| row.get::<_, rusqlite::types::Value>(index))
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()
|
||||||
|
})?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<_>>>()
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let before = rows("workspace-a");
|
||||||
|
let other = rows("workspace-b");
|
||||||
|
assert!(before.iter().all(|table| table.len() == 1));
|
||||||
|
let preflight = store
|
||||||
|
.workspace_deletion_preflight(&owner, &workspace_id)
|
||||||
|
.unwrap();
|
||||||
|
let operation = store
|
||||||
|
.reserve_workspace_deletion(
|
||||||
|
&owner,
|
||||||
|
&workspace_id,
|
||||||
|
&WorkspaceDeletionRequest {
|
||||||
|
operation_id: "purge-subject-snapshots".into(),
|
||||||
|
expected_workspace_updated_at: preflight.expected_workspace_updated_at,
|
||||||
|
confirmation: "Alpha".into(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.operation;
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch(
|
||||||
|
"CREATE TRIGGER reject_subject_snapshot_purge BEFORE DELETE ON workspaces
|
||||||
|
BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.finalize_workspace_deletion(&operation.operation_id)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
rows("workspace-a"),
|
||||||
|
before,
|
||||||
|
"rollback restores snapshots and canonical receipt"
|
||||||
|
);
|
||||||
|
assert_eq!(rows("workspace-b"), other);
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("DROP TRIGGER reject_subject_snapshot_purge")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let completed = store
|
||||||
|
.finalize_workspace_deletion(&operation.operation_id)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(completed.state, WorkspaceDeletionState::Succeeded);
|
||||||
|
assert!(rows("workspace-a").iter().all(|table| table.is_empty()));
|
||||||
|
assert_eq!(rows("workspace-b"), other);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.finalize_workspace_deletion(&operation.operation_id)
|
||||||
|
.unwrap(),
|
||||||
|
completed
|
||||||
|
);
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
assert_eq!(
|
||||||
|
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
|
||||||
|
row.get::<_, i64>(0)
|
||||||
|
})?,
|
||||||
|
0
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn deletion_is_idempotent_and_removes_workspace_scoped_rows() {
|
fn deletion_is_idempotent_and_removes_workspace_scoped_rows() {
|
||||||
let (store, owner, workspace_id) = setup();
|
let (store, owner, workspace_id) = setup();
|
||||||
|
|||||||
+169
-114
@@ -11127,6 +11127,40 @@
|
|||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
|
"SubjektivSubjectDeleteRequest": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"operation_id": {
|
||||||
|
"description": "Stable identity retained for every retry, including unknown outcomes.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"operation_id"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"SubjektivSubjectDeleteResponse": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"completed": {
|
||||||
|
"description": "False confirms the deletion fence, not external stop or erasure completion.",
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"operation_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"subject_id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"subject_id",
|
||||||
|
"operation_id",
|
||||||
|
"completed"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
"SubjektivSubjectListQuery": {
|
"SubjektivSubjectListQuery": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
@@ -11228,11 +11262,19 @@
|
|||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
"SubjektivSubjectState": {
|
"SubjektivSubjectState": {
|
||||||
"enum": [
|
"oneOf": [
|
||||||
"active",
|
{
|
||||||
"retired"
|
"enum": [
|
||||||
],
|
"active"
|
||||||
"type": "string"
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"const": "retired",
|
||||||
|
"description": "Saved compatibility only; not a current lifecycle operation.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"SubjektivWorkerConnectionRequest": {
|
"SubjektivWorkerConnectionRequest": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
@@ -17129,6 +17171,10 @@
|
|||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"description": "Backend-authoritative permissions for the current Workspace actor.",
|
"description": "Backend-authoritative permissions for the current Workspace actor.",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"delete_subjects": {
|
||||||
|
"description": "Explicit Subject deletion authority for the current authenticated actor.",
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
"delete_workspace": {
|
"delete_workspace": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
@@ -17146,7 +17192,8 @@
|
|||||||
"manage_repositories",
|
"manage_repositories",
|
||||||
"manage_secrets",
|
"manage_secrets",
|
||||||
"manage_runtimes",
|
"manage_runtimes",
|
||||||
"delete_workspace"
|
"delete_workspace",
|
||||||
|
"delete_subjects"
|
||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
@@ -18302,7 +18349,7 @@
|
|||||||
"info": {
|
"info": {
|
||||||
"title": "Yoi Server API",
|
"title": "Yoi Server API",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"x-yoi-source-digest": "sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4"
|
"x-yoi-source-digest": "sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e"
|
||||||
},
|
},
|
||||||
"openapi": "3.1.0",
|
"openapi": "3.1.0",
|
||||||
"paths": {
|
"paths": {
|
||||||
@@ -35014,6 +35061,121 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}": {
|
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}": {
|
||||||
|
"delete": {
|
||||||
|
"operationId": "subjektiv_subject_delete",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"explode": false,
|
||||||
|
"in": "path",
|
||||||
|
"name": "workspace_id",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/string"
|
||||||
|
},
|
||||||
|
"style": "simple"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"explode": false,
|
||||||
|
"in": "path",
|
||||||
|
"name": "subject_id",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/string"
|
||||||
|
},
|
||||||
|
"style": "simple"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"requestBody": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/SubjektivSubjectDeleteRequest"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/SubjektivSubjectDeleteResponse"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful response"
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"401": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"403": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"404": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"409": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"bearerAuth": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"browserSession": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"get": {
|
"get": {
|
||||||
"operationId": "subjektiv_subject_get",
|
"operationId": "subjektiv_subject_get",
|
||||||
"parameters": [
|
"parameters": [
|
||||||
@@ -35790,113 +35952,6 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire": {
|
|
||||||
"post": {
|
|
||||||
"operationId": "subjektiv_subject_retire",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"explode": false,
|
|
||||||
"in": "path",
|
|
||||||
"name": "workspace_id",
|
|
||||||
"required": true,
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/string"
|
|
||||||
},
|
|
||||||
"style": "simple"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"explode": false,
|
|
||||||
"in": "path",
|
|
||||||
"name": "subject_id",
|
|
||||||
"required": true,
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/string"
|
|
||||||
},
|
|
||||||
"style": "simple"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/SubjektivSubjectResponse"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"description": "Successful response"
|
|
||||||
},
|
|
||||||
"400": {
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/RepositoryApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"description": "Error response"
|
|
||||||
},
|
|
||||||
"401": {
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/RepositoryApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"description": "Error response"
|
|
||||||
},
|
|
||||||
"403": {
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/RepositoryApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"description": "Error response"
|
|
||||||
},
|
|
||||||
"404": {
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/RepositoryApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"description": "Error response"
|
|
||||||
},
|
|
||||||
"409": {
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/RepositoryApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"description": "Error response"
|
|
||||||
},
|
|
||||||
"500": {
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/RepositoryApiError"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"description": "Error response"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"security": [
|
|
||||||
{
|
|
||||||
"bearerAuth": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"browserSession": []
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/surface": {
|
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/surface": {
|
||||||
"get": {
|
"get": {
|
||||||
"operationId": "subjektiv_resident_surface",
|
"operationId": "subjektiv_resident_surface",
|
||||||
|
|||||||
@@ -32,6 +32,14 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "crates/subjektiv/src/deletion/tests.rs",
|
||||||
|
"kind": "saved_data",
|
||||||
|
"reason": "schema 6の固定保存Subject行を投入し、最新migrationが旧retiredとarchive原文を黙って消去・再活性化せず、明示削除だけで消去する検証に必要な旧SQL列名。現行writerや競合制御には使わない。",
|
||||||
|
"lines": [
|
||||||
|
{ "sha256": "defca5951b3d48785514b25912044fa3fc252659311b09a31fa2e64af2a19049", "count": 1 }
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "crates/subjektiv/src/journal/memory_links/tests.rs",
|
"path": "crates/subjektiv/src/journal/memory_links/tests.rs",
|
||||||
"kind": "saved_data",
|
"kind": "saved_data",
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
{
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"id": "subject-delete-dark",
|
||||||
|
"title": "Subject deletion confirmation and retry",
|
||||||
|
"baseUrl": "http://127.0.0.1:15287",
|
||||||
|
"colorScheme": "dark",
|
||||||
|
"reducedMotion": "reduce",
|
||||||
|
"personas": [
|
||||||
|
{
|
||||||
|
"id": "fixture",
|
||||||
|
"label": "Authenticated owner (synthetic HTTP)",
|
||||||
|
"auth": {
|
||||||
|
"kind": "anonymous"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"viewports": [
|
||||||
|
{
|
||||||
|
"label": "desktop",
|
||||||
|
"width": 1440,
|
||||||
|
"height": 900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "narrow",
|
||||||
|
"width": 768,
|
||||||
|
"height": 900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "mobile",
|
||||||
|
"width": 390,
|
||||||
|
"height": 844
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"routes": [
|
||||||
|
{
|
||||||
|
"id": "subject",
|
||||||
|
"label": "Subject detail",
|
||||||
|
"path": "/w/memory-review/memory/pending-subject",
|
||||||
|
"goal": "Review the Subject before irreversible deletion.",
|
||||||
|
"dataState": "Active Subject, no current Worker; owner permissions from Backend projection.",
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": ".subject-overview"
|
||||||
|
},
|
||||||
|
"capturePoints": [
|
||||||
|
{
|
||||||
|
"id": "initial",
|
||||||
|
"label": "Subject detail"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "confirmation",
|
||||||
|
"label": "Irreversible deletion scope and confirmation",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-delete-open"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=confirming]"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "pending",
|
||||||
|
"label": "Fence saved; erasure unconfirmed",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".delete-confirm input"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-deletion button[type=submit]"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=pending]"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "completed",
|
||||||
|
"label": "Deletion confirmed",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-delete-retry"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=completed]"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "legacy",
|
||||||
|
"label": "Legacy ended Subject",
|
||||||
|
"path": "/w/memory-review/memory/legacy-subject",
|
||||||
|
"goal": "Delete a legacy ended Subject without reactivating it.",
|
||||||
|
"dataState": "Saved retired wire state, owner.",
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": ".subject-delete-open"
|
||||||
|
},
|
||||||
|
"capturePoints": [
|
||||||
|
{
|
||||||
|
"id": "initial",
|
||||||
|
"label": "Legacy ended Subject"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "confirmation",
|
||||||
|
"label": "Delete legacy Subject",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-delete-open"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=confirming]"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "non-owner",
|
||||||
|
"label": "Nonowner Subject view",
|
||||||
|
"path": "/w/memory-review/memory/legacy-subject?persona=reader",
|
||||||
|
"goal": "Read Subject without presenting unauthorized deletion.",
|
||||||
|
"dataState": "Backend delete_subjects=false; no deletion controls.",
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": ".subject-overview"
|
||||||
|
},
|
||||||
|
"capturePoints": [
|
||||||
|
{
|
||||||
|
"id": "initial",
|
||||||
|
"label": "Nonowner details"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"processes": [
|
||||||
|
{
|
||||||
|
"id": "subject-delete-fixture",
|
||||||
|
"command": "deno",
|
||||||
|
"cwd": "..",
|
||||||
|
"args": [
|
||||||
|
"run",
|
||||||
|
"-A",
|
||||||
|
"../../web/workspace/test/subject-delete-browser/fixture-server.ts",
|
||||||
|
"15287",
|
||||||
|
"../../web/workspace/build"
|
||||||
|
],
|
||||||
|
"readyUrl": "http://127.0.0.1:15287/health"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
{
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"id": "subject-delete-light",
|
||||||
|
"title": "Subject deletion confirmation and retry",
|
||||||
|
"baseUrl": "http://127.0.0.1:15287",
|
||||||
|
"colorScheme": "light",
|
||||||
|
"reducedMotion": "reduce",
|
||||||
|
"personas": [
|
||||||
|
{
|
||||||
|
"id": "fixture",
|
||||||
|
"label": "Authenticated owner (synthetic HTTP)",
|
||||||
|
"auth": {
|
||||||
|
"kind": "anonymous"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"viewports": [
|
||||||
|
{
|
||||||
|
"label": "desktop",
|
||||||
|
"width": 1440,
|
||||||
|
"height": 900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "narrow",
|
||||||
|
"width": 768,
|
||||||
|
"height": 900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "mobile",
|
||||||
|
"width": 390,
|
||||||
|
"height": 844
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"routes": [
|
||||||
|
{
|
||||||
|
"id": "subject",
|
||||||
|
"label": "Subject detail",
|
||||||
|
"path": "/w/memory-review/memory/pending-subject",
|
||||||
|
"goal": "Review the Subject before irreversible deletion.",
|
||||||
|
"dataState": "Active Subject, no current Worker; owner permissions from Backend projection.",
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": ".subject-overview"
|
||||||
|
},
|
||||||
|
"capturePoints": [
|
||||||
|
{
|
||||||
|
"id": "initial",
|
||||||
|
"label": "Subject detail"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "confirmation",
|
||||||
|
"label": "Irreversible deletion scope and confirmation",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-delete-open"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=confirming]"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "pending",
|
||||||
|
"label": "Fence saved; erasure unconfirmed",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".delete-confirm input"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-deletion button[type=submit]"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=pending]"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "completed",
|
||||||
|
"label": "Deletion confirmed",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-delete-retry"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=completed]"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "legacy",
|
||||||
|
"label": "Legacy ended Subject",
|
||||||
|
"path": "/w/memory-review/memory/legacy-subject",
|
||||||
|
"goal": "Delete a legacy ended Subject without reactivating it.",
|
||||||
|
"dataState": "Saved retired wire state, owner.",
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": ".subject-delete-open"
|
||||||
|
},
|
||||||
|
"capturePoints": [
|
||||||
|
{
|
||||||
|
"id": "initial",
|
||||||
|
"label": "Legacy ended Subject"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "confirmation",
|
||||||
|
"label": "Delete legacy Subject",
|
||||||
|
"interaction": [
|
||||||
|
{
|
||||||
|
"action": "click",
|
||||||
|
"selector": ".subject-delete-open"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": "[data-delete-state=confirming]"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "non-owner",
|
||||||
|
"label": "Nonowner Subject view",
|
||||||
|
"path": "/w/memory-review/memory/legacy-subject?persona=reader",
|
||||||
|
"goal": "Read Subject without presenting unauthorized deletion.",
|
||||||
|
"dataState": "Backend delete_subjects=false; no deletion controls.",
|
||||||
|
"ready": {
|
||||||
|
"kind": "selector",
|
||||||
|
"selector": ".subject-overview"
|
||||||
|
},
|
||||||
|
"capturePoints": [
|
||||||
|
{
|
||||||
|
"id": "initial",
|
||||||
|
"label": "Nonowner details"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"processes": [
|
||||||
|
{
|
||||||
|
"id": "subject-delete-fixture",
|
||||||
|
"command": "deno",
|
||||||
|
"cwd": "..",
|
||||||
|
"args": [
|
||||||
|
"run",
|
||||||
|
"-A",
|
||||||
|
"../../web/workspace/test/subject-delete-browser/fixture-server.ts",
|
||||||
|
"15287",
|
||||||
|
"../../web/workspace/build"
|
||||||
|
],
|
||||||
|
"readyUrl": "http://127.0.0.1:15287/health"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@
|
|||||||
"dev": "deno run -A npm:vite@8.3.2 dev",
|
"dev": "deno run -A npm:vite@8.3.2 dev",
|
||||||
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
||||||
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
|
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
|
||||||
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/subject-delete.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
||||||
"test:journal:backend-contract": "cd ../.. && mkdir -p target && cargo test -p yoi-workspace-server --lib journal_browser_web_contract -- --nocapture > target/journal-browser-contract.log && cd web/workspace && deno run --allow-read=../../target/journal-browser-contract.log test/journal-backend-contract.ts ../../target/journal-browser-contract.log",
|
"test:journal:backend-contract": "cd ../.. && mkdir -p target && cargo test -p yoi-workspace-server --lib journal_browser_web_contract -- --nocapture > target/journal-browser-contract.log && cd web/workspace && deno run --allow-read=../../target/journal-browser-contract.log test/journal-backend-contract.ts ../../target/journal-browser-contract.log",
|
||||||
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
|
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
|
||||||
"test:browser:activation": "mkdir -p ../../target/web-ux/tmp && TMPDIR=../../target/web-ux/tmp deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/activation-browser/activation-shell.browser.test.ts",
|
"test:browser:activation": "mkdir -p ../../target/web-ux/tmp && TMPDIR=../../target/web-ux/tmp deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/activation-browser/activation-shell.browser.test.ts",
|
||||||
|
|||||||
Generated
+7
@@ -3,6 +3,7 @@
|
|||||||
"specifiers": {
|
"specifiers": {
|
||||||
"jsr:@std/assert@*": "1.0.19",
|
"jsr:@std/assert@*": "1.0.19",
|
||||||
"jsr:@std/internal@^1.0.12": "1.0.14",
|
"jsr:@std/internal@^1.0.12": "1.0.14",
|
||||||
|
"jsr:@std/path@1.1.4": "1.1.4",
|
||||||
"npm:@codemirror/autocomplete@6.20.3": "6.20.3",
|
"npm:@codemirror/autocomplete@6.20.3": "6.20.3",
|
||||||
"npm:@codemirror/commands@6.11.1": "6.11.1",
|
"npm:@codemirror/commands@6.11.1": "6.11.1",
|
||||||
"npm:@codemirror/language@6.12.4": "6.12.4",
|
"npm:@codemirror/language@6.12.4": "6.12.4",
|
||||||
@@ -42,6 +43,12 @@
|
|||||||
},
|
},
|
||||||
"@std/internal@1.0.14": {
|
"@std/internal@1.0.14": {
|
||||||
"integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7"
|
"integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7"
|
||||||
|
},
|
||||||
|
"@std/path@1.1.4": {
|
||||||
|
"integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5",
|
||||||
|
"dependencies": [
|
||||||
|
"jsr:@std/internal"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"npm": {
|
"npm": {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
|
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||||
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/companion-api.ts
|
// Output: web/workspace/src/lib/generated/companion-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
|
||||||
|
|||||||
@@ -52,6 +52,10 @@ export type WorkspacePermissionSummary = {
|
|||||||
manage_secrets: boolean;
|
manage_secrets: boolean;
|
||||||
manage_runtimes: boolean;
|
manage_runtimes: boolean;
|
||||||
delete_workspace: boolean;
|
delete_workspace: boolean;
|
||||||
|
/**
|
||||||
|
* Explicit Subject deletion authority for the current authenticated actor.
|
||||||
|
*/
|
||||||
|
delete_subjects: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type WorkspaceDeletionState =
|
export type WorkspaceDeletionState =
|
||||||
|
|||||||
@@ -118,6 +118,14 @@ export type SubjektivSubjectCreateRequest = {
|
|||||||
behavior_md?: string;
|
behavior_md?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type SubjektivSubjectDeleteRequest = { operation_id: string };
|
||||||
|
|
||||||
|
export type SubjektivSubjectDeleteResponse = {
|
||||||
|
subject_id: string;
|
||||||
|
operation_id: string;
|
||||||
|
completed: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export type SubjektivSubjectBehaviorUpdateRequest = {
|
export type SubjektivSubjectBehaviorUpdateRequest = {
|
||||||
expected_behavior_md: string;
|
expected_behavior_md: string;
|
||||||
behavior_md: string;
|
behavior_md: string;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
|
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||||
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
|
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/repository-api.ts
|
// Output: web/workspace/src/lib/generated/repository-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
|
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||||
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/runtime-api.ts
|
// Output: web/workspace/src/lib/generated/runtime-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
|
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||||
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/ticket-api.ts
|
// Output: web/workspace/src/lib/generated/ticket-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
|
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
|
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
|
||||||
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
|
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
|
||||||
|
|||||||
@@ -482,6 +482,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary {
|
|||||||
"manage_secrets",
|
"manage_secrets",
|
||||||
"manage_runtimes",
|
"manage_runtimes",
|
||||||
"delete_workspace",
|
"delete_workspace",
|
||||||
|
"delete_subjects",
|
||||||
],
|
],
|
||||||
path,
|
path,
|
||||||
);
|
);
|
||||||
@@ -492,6 +493,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary {
|
|||||||
),
|
),
|
||||||
manage_secrets: boolean(item.manage_secrets, `${path}.manage_secrets`),
|
manage_secrets: boolean(item.manage_secrets, `${path}.manage_secrets`),
|
||||||
manage_runtimes: boolean(item.manage_runtimes, `${path}.manage_runtimes`),
|
manage_runtimes: boolean(item.manage_runtimes, `${path}.manage_runtimes`),
|
||||||
|
delete_subjects: boolean(item.delete_subjects, `${path}.delete_subjects`),
|
||||||
delete_workspace: boolean(
|
delete_workspace: boolean(
|
||||||
item.delete_workspace,
|
item.delete_workspace,
|
||||||
`${path}.delete_workspace`,
|
`${path}.delete_workspace`,
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { untrack } from 'svelte';
|
||||||
|
import { deleteSubjektivSubject, SubjektivSubjectDeleteError } from './api.ts';
|
||||||
|
import { clearSubjectDeleteAttempt, readSubjectDeleteAttempt, saveSubjectDeleteAttempt, type SubjectDeleteAttempt } from './subject-delete.ts';
|
||||||
|
import type { SubjektivSubjectResponse } from '#lib/generated/memory-api.ts';
|
||||||
|
import { workspaceRoute } from '#lib/workspace/api/http.ts';
|
||||||
|
|
||||||
|
let { workspaceId, subjectId, subject, onstarted }: {
|
||||||
|
workspaceId: string;
|
||||||
|
subjectId: string;
|
||||||
|
subject: SubjektivSubjectResponse | null;
|
||||||
|
onstarted: () => void;
|
||||||
|
} = $props();
|
||||||
|
let attempt = $state<SubjectDeleteAttempt | null>(null);
|
||||||
|
let phase = $state<'idle' | 'confirming' | 'running' | 'pending' | 'unknown' | 'failed' | 'completed'>('idle');
|
||||||
|
let confirmed = $state(false);
|
||||||
|
let error = $state<string | null>(null);
|
||||||
|
let storageError = $state(false);
|
||||||
|
let identity = '';
|
||||||
|
let generation = 0;
|
||||||
|
|
||||||
|
$effect(() => {
|
||||||
|
const next = JSON.stringify([workspaceId, subjectId]);
|
||||||
|
untrack(() => {
|
||||||
|
if (identity === next) return;
|
||||||
|
identity = next;
|
||||||
|
generation += 1;
|
||||||
|
attempt = null;
|
||||||
|
phase = 'idle';
|
||||||
|
confirmed = false;
|
||||||
|
error = null;
|
||||||
|
storageError = false;
|
||||||
|
try {
|
||||||
|
attempt = readSubjectDeleteAttempt(sessionStorage, workspaceId, subjectId);
|
||||||
|
if (attempt) {
|
||||||
|
phase = 'unknown';
|
||||||
|
onstarted();
|
||||||
|
}
|
||||||
|
} catch (cause) {
|
||||||
|
storageError = true;
|
||||||
|
error = cause instanceof Error ? cause.message : 'Cannot read the saved deletion request. Do not start a new request.';
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
async function submit(): Promise<void> {
|
||||||
|
if (phase === 'running' || phase === 'completed' || storageError || (!attempt && (!subject || !confirmed))) return;
|
||||||
|
const currentGeneration = generation;
|
||||||
|
const targetWorkspace = workspaceId;
|
||||||
|
const targetSubject = subjectId;
|
||||||
|
const request = attempt ?? { subjectId: targetSubject, operationId: crypto.randomUUID() };
|
||||||
|
const current = () => currentGeneration === generation && targetWorkspace === workspaceId && targetSubject === subjectId;
|
||||||
|
// 保存に失敗したら送らない。応答喪失後に新しいkeyを作る経路を残さない。
|
||||||
|
try { saveSubjectDeleteAttempt(sessionStorage, targetWorkspace, request); }
|
||||||
|
catch {
|
||||||
|
error = 'The retry identity could not be saved in this tab. No deletion request was sent. Allow session storage and try again.';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
attempt = request;
|
||||||
|
phase = 'running';
|
||||||
|
error = null;
|
||||||
|
onstarted();
|
||||||
|
try {
|
||||||
|
const result = await deleteSubjektivSubject(fetch, targetWorkspace, targetSubject, { operation_id: request.operationId });
|
||||||
|
// 対象変更後の遅延応答は新対象を更新せず、成功した旧対象の再送入力だけ消去する。
|
||||||
|
if (result.completed) {
|
||||||
|
try { clearSubjectDeleteAttempt(sessionStorage, targetWorkspace, targetSubject); }
|
||||||
|
catch { /* 消去できなくても残るのは同一keyだけ。本文や成功状態は保存しない。 */ }
|
||||||
|
}
|
||||||
|
if (!current()) return;
|
||||||
|
phase = result.completed ? 'completed' : 'pending';
|
||||||
|
} catch (cause) {
|
||||||
|
if (!current()) return;
|
||||||
|
phase = cause instanceof SubjektivSubjectDeleteError && cause.kind === 'rejected' ? 'failed' : 'unknown';
|
||||||
|
error = cause instanceof Error ? cause.message : 'The deletion outcome is unknown.';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
{#if subject || attempt || storageError}
|
||||||
|
<section class="subject-deletion" aria-labelledby="subject-delete-heading" data-delete-state={phase}>
|
||||||
|
<header>
|
||||||
|
<h2 id="subject-delete-heading">Delete Subject</h2>
|
||||||
|
{#if phase === 'idle' && !storageError}
|
||||||
|
<button type="button" class="workspace-secondary-button subject-delete-open" onclick={() => { phase = 'confirming'; }}>Delete Subject…</button>
|
||||||
|
{/if}
|
||||||
|
</header>
|
||||||
|
{#if phase === 'confirming' || attempt}
|
||||||
|
<p class="delete-target">Target: <strong>{subject?.role ?? subjectId}</strong> · <code>{subjectId}</code></p>
|
||||||
|
{/if}
|
||||||
|
{#if phase === 'confirming'}
|
||||||
|
<form onsubmit={(event) => { event.preventDefault(); void submit(); }}>
|
||||||
|
<p><strong>This is irreversible.</strong> This Subject, its behavior, Memory, change history, staging, Surface, Journal, permissions, delegations, Writer/Session associations, and Subject Job inputs/results will be deleted. Connected and delegated Workers and Jobs must stop before completion can be confirmed.</p>
|
||||||
|
<p>Original Worker/Session text, existing archives and Workdirs, Tickets, and other separately retained resources are <strong>not deleted</strong>. Minimal deletion retry/audit metadata is retained. Other Subjects and Workspaces are not affected.</p>
|
||||||
|
<label class="delete-confirm"><input type="checkbox" bind:checked={confirmed} /> I have checked the target, deletion scope, retained data, and irreversible effect.</label>
|
||||||
|
<div class="delete-actions">
|
||||||
|
<button type="submit" class="workspace-danger-button" disabled={!confirmed}>Permanently delete Subject</button>
|
||||||
|
<button type="button" class="workspace-secondary-button" onclick={() => { phase = 'idle'; confirmed = false; }}>Cancel</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
{:else if phase === 'running'}
|
||||||
|
<p role="status">Deleting… Waiting for stop and erasure confirmation. Do not treat this as completed.</p>
|
||||||
|
{:else if phase === 'pending'}
|
||||||
|
<p role="status"><strong>Deletion started; completion unconfirmed.</strong> The deletion fence is saved and the Subject is hidden from reads/lists. External stopping or erasure is not yet confirmed. Retry the same request to continue.</p>
|
||||||
|
{:else if phase === 'unknown'}
|
||||||
|
<p role="status"><strong>Deletion outcome unknown.</strong> The request may already have taken effect. The saved target and operation ID are retained; retry the same request, not a new deletion.</p>
|
||||||
|
{:else if phase === 'failed'}
|
||||||
|
<p role="alert"><strong>Deletion request rejected.</strong> Completion is not confirmed. Resolve the reported condition, then retry the same request.</p>
|
||||||
|
{:else if phase === 'completed'}
|
||||||
|
<p role="status"><strong>Subject deletion completed.</strong> Backend confirmed stopping and deletion of Subject-owned data. Separately retained Worker/Session text, archives, Workdirs, and Tickets remain.</p>
|
||||||
|
<a href={workspaceRoute(workspaceId, '/memory')}>Back to subjects →</a>
|
||||||
|
{/if}
|
||||||
|
{#if error}<p class="workspace-callout is-error" role="alert">{error}</p>{/if}
|
||||||
|
{#if attempt && phase !== 'completed'}
|
||||||
|
<details><summary>Deletion request identity</summary><code>{attempt.operationId}</code><p>This tab retains the target and same request ID across reloads, even when GET can no longer find the Subject.</p></details>
|
||||||
|
{#if phase === 'pending' || phase === 'unknown' || phase === 'failed'}
|
||||||
|
<button type="button" class="workspace-secondary-button subject-delete-retry" onclick={() => { void submit(); }}>Retry same deletion request</button>
|
||||||
|
{/if}
|
||||||
|
{/if}
|
||||||
|
</section>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.subject-deletion { min-width: 0; }
|
||||||
|
header { display: flex; align-items: center; justify-content: space-between; gap: .75rem; flex-wrap: wrap; }
|
||||||
|
h2 { font-size: var(--font-size-body); margin: 0; }
|
||||||
|
p { line-height: 1.6; }
|
||||||
|
code, .delete-target { overflow-wrap: anywhere; }
|
||||||
|
.delete-confirm { display: flex; align-items: flex-start; gap: .6rem; line-height: 1.5; }
|
||||||
|
input { margin-top: .25rem; flex: 0 0 auto; }
|
||||||
|
.delete-actions { display: flex; gap: .75rem; flex-wrap: wrap; margin-top: 1rem; }
|
||||||
|
details { margin-block: 1rem; }
|
||||||
|
summary { cursor: pointer; }
|
||||||
|
</style>
|
||||||
@@ -29,6 +29,8 @@ import type {
|
|||||||
SubjektivResidentSurfaceAvailability,
|
SubjektivResidentSurfaceAvailability,
|
||||||
SubjektivResidentSurfaceResponse,
|
SubjektivResidentSurfaceResponse,
|
||||||
SubjektivResidentSurfaceSnapshot,
|
SubjektivResidentSurfaceSnapshot,
|
||||||
|
SubjektivSubjectDeleteRequest,
|
||||||
|
SubjektivSubjectDeleteResponse,
|
||||||
SubjektivSubjectBehaviorUpdateRequest,
|
SubjektivSubjectBehaviorUpdateRequest,
|
||||||
SubjektivSubjectCreateRequest,
|
SubjektivSubjectCreateRequest,
|
||||||
SubjektivSubjectListResponse,
|
SubjektivSubjectListResponse,
|
||||||
@@ -158,6 +160,70 @@ function validateSubjectBehavior(behaviorMd: string): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class SubjektivSubjectDeleteError extends Error {
|
||||||
|
constructor(
|
||||||
|
readonly kind: "rejected" | "unknown_outcome",
|
||||||
|
message: string,
|
||||||
|
readonly status: number | null = null,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "SubjektivSubjectDeleteError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseSubjektivSubjectDeleteResponse(
|
||||||
|
value: unknown,
|
||||||
|
subjectId: string,
|
||||||
|
operationId: string,
|
||||||
|
): SubjektivSubjectDeleteResponse {
|
||||||
|
const record = strictRecord(value, ["subject_id", "operation_id", "completed"], "Subject deletion");
|
||||||
|
const response = {
|
||||||
|
subject_id: requiredIdentifier(record, "subject_id"),
|
||||||
|
operation_id: requiredIdentifier(record, "operation_id"),
|
||||||
|
completed: requiredBoolean(record, "completed"),
|
||||||
|
};
|
||||||
|
if (response.subject_id !== subjectId || response.operation_id !== operationId) {
|
||||||
|
invalid("Subject deletion response does not match the request");
|
||||||
|
}
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** HTTP success can confirm only the deletion fence. Every retry retains this request. */
|
||||||
|
export async function deleteSubjektivSubject(
|
||||||
|
fetchFn: typeof fetch,
|
||||||
|
workspaceId: string,
|
||||||
|
subjectId: string,
|
||||||
|
request: SubjektivSubjectDeleteRequest,
|
||||||
|
): Promise<SubjektivSubjectDeleteResponse> {
|
||||||
|
identifier(request.operation_id, "operation_id");
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetchFn(workspaceApiPath(workspaceId, `/subjektiv/subjects/${encodeURIComponent(subjectId)}`), {
|
||||||
|
method: "DELETE",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify({ operation_id: request.operation_id }),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion outcome is unknown. Retry the same request.");
|
||||||
|
}
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new SubjektivSubjectDeleteError(
|
||||||
|
response.status >= 500 ? "unknown_outcome" : "rejected",
|
||||||
|
await subjektivSubjectCreateErrorMessage(response),
|
||||||
|
response.status,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return parseSubjektivSubjectDeleteResponse(
|
||||||
|
await readBoundedJson(response, MEMORY_API_LIMITS.maxResponseBytes),
|
||||||
|
subjectId,
|
||||||
|
request.operation_id,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion response could not be confirmed. Retry the same request.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function createSubjektivSubject(
|
export async function createSubjektivSubject(
|
||||||
fetchFn: typeof fetch,
|
fetchFn: typeof fetch,
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
// 削除開始後はGETから対象が消えるため、同じtabでの再読込みにも依頼IDを保持する。
|
||||||
|
// Subject本文は保存しない。これは削除結果の正本ではなく、安全な再送用の最小入力。
|
||||||
|
export type SubjectDeleteAttempt = {
|
||||||
|
subjectId: string;
|
||||||
|
operationId: string;
|
||||||
|
};
|
||||||
|
export function subjectDeleteStorageKey(
|
||||||
|
workspaceId: string,
|
||||||
|
subjectId: string,
|
||||||
|
): string {
|
||||||
|
return `yoi.subject-delete:${JSON.stringify([workspaceId, subjectId])}`;
|
||||||
|
}
|
||||||
|
export function readSubjectDeleteAttempt(
|
||||||
|
storage: Pick<Storage, "getItem">,
|
||||||
|
workspaceId: string,
|
||||||
|
subjectId: string,
|
||||||
|
): SubjectDeleteAttempt | null {
|
||||||
|
const raw = storage.getItem(subjectDeleteStorageKey(workspaceId, subjectId));
|
||||||
|
if (raw === null) return null;
|
||||||
|
const value = JSON.parse(raw);
|
||||||
|
if (
|
||||||
|
!value || value.subjectId !== subjectId ||
|
||||||
|
typeof value.operationId !== "string" ||
|
||||||
|
value.operationId.length === 0 ||
|
||||||
|
Object.keys(value).some((key) =>
|
||||||
|
key !== "subjectId" && key !== "operationId"
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
"The saved deletion request cannot be read. Do not start another deletion request.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
export function saveSubjectDeleteAttempt(
|
||||||
|
storage: Pick<Storage, "setItem">,
|
||||||
|
workspaceId: string,
|
||||||
|
attempt: SubjectDeleteAttempt,
|
||||||
|
): void {
|
||||||
|
storage.setItem(
|
||||||
|
subjectDeleteStorageKey(workspaceId, attempt.subjectId),
|
||||||
|
JSON.stringify(attempt),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
export function clearSubjectDeleteAttempt(
|
||||||
|
storage: Pick<Storage, "removeItem">,
|
||||||
|
workspaceId: string,
|
||||||
|
subjectId: string,
|
||||||
|
): void {
|
||||||
|
storage.removeItem(subjectDeleteStorageKey(workspaceId, subjectId));
|
||||||
|
}
|
||||||
@@ -38,7 +38,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
function subjectStateLabel(state: SubjektivSubjectResponse['state']): string {
|
function subjectStateLabel(state: SubjektivSubjectResponse['state']): string {
|
||||||
return state === 'active' ? 'Available' : 'Retired';
|
return state === 'active' ? 'Available' : 'Legacy ended Subject';
|
||||||
}
|
}
|
||||||
|
|
||||||
async function openCreateForm(): Promise<void> {
|
async function openCreateForm(): Promise<void> {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
// 実行中Workerの履歴へ反映済みであることの証明には使わない。診断値は詳細へ分離する。
|
// 実行中Workerの履歴へ反映済みであることの証明には使わない。診断値は詳細へ分離する。
|
||||||
import { untrack } from 'svelte';
|
import { untrack } from 'svelte';
|
||||||
import DocumentMarkdown from '#lib/workspace/markdown/DocumentMarkdown.svelte';
|
import DocumentMarkdown from '#lib/workspace/markdown/DocumentMarkdown.svelte';
|
||||||
|
import SubjectDelete from '#lib/workspace/memory/SubjectDelete.svelte';
|
||||||
import { formatDate, workspaceRoute } from '#lib/workspace/api/http.ts';
|
import { formatDate, workspaceRoute } from '#lib/workspace/api/http.ts';
|
||||||
import {
|
import {
|
||||||
SubjektivSubjectCreateError,
|
SubjektivSubjectCreateError,
|
||||||
@@ -26,6 +27,7 @@
|
|||||||
let behaviorSaving = $state(false);
|
let behaviorSaving = $state(false);
|
||||||
let behaviorError = $state<string | null>(null);
|
let behaviorError = $state<string | null>(null);
|
||||||
let behaviorSaveConfirmed = $state(false);
|
let behaviorSaveConfirmed = $state(false);
|
||||||
|
let deletionStarted = $state(false);
|
||||||
|
|
||||||
let routeGeneration = 0;
|
let routeGeneration = 0;
|
||||||
let subjectWorkspaceId = untrack(() => data.workspaceId);
|
let subjectWorkspaceId = untrack(() => data.workspaceId);
|
||||||
@@ -49,6 +51,7 @@
|
|||||||
behaviorSaving = false;
|
behaviorSaving = false;
|
||||||
behaviorError = null;
|
behaviorError = null;
|
||||||
behaviorSaveConfirmed = false;
|
behaviorSaveConfirmed = false;
|
||||||
|
deletionStarted = false;
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -149,7 +152,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
function subjectStateLabel(state: 'active' | 'retired'): string {
|
function subjectStateLabel(state: 'active' | 'retired'): string {
|
||||||
return state === 'active' ? 'Available' : 'Retired';
|
return state === 'active' ? 'Available' : 'Legacy ended Subject';
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
@@ -167,9 +170,16 @@
|
|||||||
<h1 id="subject-heading">{subject?.role ?? data.subjectId}</h1>
|
<h1 id="subject-heading">{subject?.role ?? data.subjectId}</h1>
|
||||||
<code class="subject-id" title={data.subjectId}>{data.subjectId}</code>
|
<code class="subject-id" title={data.subjectId}>{data.subjectId}</code>
|
||||||
</div>
|
</div>
|
||||||
<span class="read-only-label">Behavior editable</span>
|
{#if !deletionStarted}<span class="read-only-label">Behavior editable</span>{/if}
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
|
{#if data.workspace.permissions.delete_subjects}
|
||||||
|
{#key JSON.stringify([data.workspaceId, data.subjectId])}
|
||||||
|
<SubjectDelete workspaceId={data.workspaceId} subjectId={data.subjectId} {subject} onstarted={() => { deletionStarted = true; }} />
|
||||||
|
{/key}
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{#if !deletionStarted}
|
||||||
<nav aria-label="Subjectの記録"><a href={workspaceRoute(data.workspaceId, `/memory/${encodeURIComponent(data.subjectId)}/journal`)}>Journal · 最近の出来事とSessionを探す →</a></nav>
|
<nav aria-label="Subjectの記録"><a href={workspaceRoute(data.workspaceId, `/memory/${encodeURIComponent(data.subjectId)}/journal`)}>Journal · 最近の出来事とSessionを探す →</a></nav>
|
||||||
|
|
||||||
{#if subject}
|
{#if subject}
|
||||||
@@ -388,6 +398,7 @@
|
|||||||
<div class="memory-state" role="status"><p>Current Memory data is unavailable.</p></div>
|
<div class="memory-state" role="status"><p>Current Memory data is unavailable.</p></div>
|
||||||
{/if}
|
{/if}
|
||||||
</section>
|
</section>
|
||||||
|
{/if}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
// 本番Web buildと合成HTTP境界を接続する。実認証・削除の永続性はBackend suiteの責務。
|
||||||
|
import { join, resolve } from "jsr:@std/path@1.1.4";
|
||||||
|
const [portText, build, mode] = Deno.args;
|
||||||
|
const port = Number(portText);
|
||||||
|
const root = resolve(import.meta.dirname!, "../../../..");
|
||||||
|
const upstreamPort = port + 1;
|
||||||
|
const upstream = new Deno.Command(Deno.execPath(), {
|
||||||
|
args: [
|
||||||
|
"run",
|
||||||
|
"-A",
|
||||||
|
join(root, "tools/web-ux/browser-tests/memory_document_fixture_server.ts"),
|
||||||
|
String(upstreamPort),
|
||||||
|
resolve(build),
|
||||||
|
"3",
|
||||||
|
],
|
||||||
|
stdout: "null",
|
||||||
|
stderr: "inherit",
|
||||||
|
}).spawn();
|
||||||
|
const stop = () => {
|
||||||
|
try {
|
||||||
|
upstream.kill();
|
||||||
|
} catch { /* already stopped */ }
|
||||||
|
};
|
||||||
|
Deno.addSignalListener("SIGTERM", () => {
|
||||||
|
stop();
|
||||||
|
Deno.exit();
|
||||||
|
});
|
||||||
|
Deno.addSignalListener("SIGINT", () => {
|
||||||
|
stop();
|
||||||
|
Deno.exit();
|
||||||
|
});
|
||||||
|
const attempts = new Map<string, string>();
|
||||||
|
const hidden = new Set<string>();
|
||||||
|
const user = {
|
||||||
|
user_id: "fixture-user",
|
||||||
|
account_id: "owner",
|
||||||
|
handle: "owner",
|
||||||
|
display_name: "Memory owner",
|
||||||
|
};
|
||||||
|
const apiRoot = "/api/w/memory-review";
|
||||||
|
Deno.serve({ hostname: "127.0.0.1", port }, async (request) => {
|
||||||
|
const url = new URL(request.url);
|
||||||
|
if (url.pathname === "/health") {
|
||||||
|
try {
|
||||||
|
return await fetch(`http://127.0.0.1:${upstreamPort}/health`);
|
||||||
|
} catch {
|
||||||
|
return new Response(null, { status: 503 });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (url.pathname === "/api/auth/whoami") {
|
||||||
|
return Response.json({
|
||||||
|
authentication_reference: "subject-delete-fixture",
|
||||||
|
actor: { ...user, auth_method: "browser_session" },
|
||||||
|
session: { kind: "authenticated", expires_at: "2099-01-01T00:00:00Z" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const sessionId =
|
||||||
|
request.headers.get("cookie")?.match(/delete-review-session=([^;]+)/)
|
||||||
|
?.[1] ?? crypto.randomUUID();
|
||||||
|
const reader =
|
||||||
|
request.headers.get("cookie")?.includes("subject-delete-persona=reader") ??
|
||||||
|
false;
|
||||||
|
const match = url.pathname.match(/\/subjektiv\/subjects\/([^/]+)(?:\/|$)/);
|
||||||
|
const id = match ? decodeURIComponent(match[1]) : null;
|
||||||
|
const key = JSON.stringify([sessionId, id]);
|
||||||
|
if (request.method === "DELETE" && id) {
|
||||||
|
if (reader) {
|
||||||
|
return Response.json({ error: "forbidden", message: "Owner required" }, {
|
||||||
|
status: 403,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const body = await request.json();
|
||||||
|
const prior = attempts.get(key);
|
||||||
|
if (prior && prior !== body.operation_id) {
|
||||||
|
return Response.json(
|
||||||
|
{ error: "conflict", message: "Changed operation" },
|
||||||
|
{ status: 409 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
attempts.set(key, body.operation_id);
|
||||||
|
hidden.add(key);
|
||||||
|
if (!prior && id === "unknown-subject") {
|
||||||
|
return Response.json({
|
||||||
|
error: "internal",
|
||||||
|
message: "Response lost after fence",
|
||||||
|
}, { status: 500 });
|
||||||
|
}
|
||||||
|
return Response.json({
|
||||||
|
subject_id: id,
|
||||||
|
operation_id: body.operation_id,
|
||||||
|
completed: !!prior,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (id && request.method === "GET" && hidden.has(key)) {
|
||||||
|
return Response.json(
|
||||||
|
{ error: "not_found", message: "Subject unavailable" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const mapped = id &&
|
||||||
|
["pending-subject", "unknown-subject", "legacy-subject"].includes(id);
|
||||||
|
const target = new URL(request.url);
|
||||||
|
target.port = String(upstreamPort);
|
||||||
|
if (mapped) {
|
||||||
|
target.pathname = target.pathname.replace(
|
||||||
|
encodeURIComponent(id!),
|
||||||
|
"empty-subject",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let response = await fetch(new Request(target, request));
|
||||||
|
if (url.pathname === `${apiRoot}/workspace` || mapped) {
|
||||||
|
const data = await response.json();
|
||||||
|
if (url.pathname === `${apiRoot}/workspace` && mode !== "before") {
|
||||||
|
data.permissions.delete_subjects = !reader;
|
||||||
|
}
|
||||||
|
if (mapped) {
|
||||||
|
if ("id" in data) {
|
||||||
|
data.id = id;
|
||||||
|
data.role = `Deletion review · ${id}`;
|
||||||
|
if (id === "legacy-subject") data.state = "retired";
|
||||||
|
}
|
||||||
|
if ("subject_id" in data) data.subject_id = id;
|
||||||
|
}
|
||||||
|
response = Response.json(data, { status: response.status });
|
||||||
|
}
|
||||||
|
response = new Response(response.body, {
|
||||||
|
status: response.status,
|
||||||
|
headers: response.headers,
|
||||||
|
});
|
||||||
|
const persona = url.searchParams.get("persona");
|
||||||
|
if (persona) {
|
||||||
|
response.headers.set(
|
||||||
|
"set-cookie",
|
||||||
|
`subject-delete-persona=${persona}; Path=/; HttpOnly; SameSite=Lax`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
response.headers.append(
|
||||||
|
"set-cookie",
|
||||||
|
`delete-review-session=${sessionId}; Path=/; HttpOnly; SameSite=Lax`,
|
||||||
|
);
|
||||||
|
return response;
|
||||||
|
});
|
||||||
@@ -0,0 +1,257 @@
|
|||||||
|
import { assert, assertEquals } from "@std/assert";
|
||||||
|
import { chromium, type Page } from "playwright";
|
||||||
|
import { join, resolve } from "@std/path";
|
||||||
|
import {
|
||||||
|
assertNoPageWideOverflow,
|
||||||
|
freePort,
|
||||||
|
waitForServer,
|
||||||
|
} from "../../../../tools/web-ux/browser-tests/memory_document_harness.ts";
|
||||||
|
|
||||||
|
const root = resolve(import.meta.dirname!, "../../../..");
|
||||||
|
const artifact = join(root, "target/web-ux/t786-browser");
|
||||||
|
async function confirm(page: Page) {
|
||||||
|
await page.locator(".subject-delete-open").click();
|
||||||
|
const button = page.getByRole("button", {
|
||||||
|
name: "Permanently delete Subject",
|
||||||
|
exact: true,
|
||||||
|
});
|
||||||
|
assert(await button.isDisabled());
|
||||||
|
await page.getByRole("checkbox").check();
|
||||||
|
await button.click();
|
||||||
|
}
|
||||||
|
Deno.test({
|
||||||
|
name:
|
||||||
|
"Subject deletion shell: owner confirmation, pending/reload/retry, unknown/rejected and route fencing",
|
||||||
|
sanitizeOps: false,
|
||||||
|
sanitizeResources: false,
|
||||||
|
fn: async () => {
|
||||||
|
const build = await new Deno.Command(Deno.execPath(), {
|
||||||
|
args: ["task", "build"],
|
||||||
|
cwd: join(root, "web/workspace"),
|
||||||
|
stdout: "null",
|
||||||
|
stderr: "piped",
|
||||||
|
}).output();
|
||||||
|
assert(build.success, new TextDecoder().decode(build.stderr));
|
||||||
|
const port = await freePort();
|
||||||
|
const server = new Deno.Command(Deno.execPath(), {
|
||||||
|
args: [
|
||||||
|
"run",
|
||||||
|
"-A",
|
||||||
|
join(import.meta.dirname!, "fixture-server.ts"),
|
||||||
|
String(port),
|
||||||
|
join(root, "web/workspace/build"),
|
||||||
|
],
|
||||||
|
stdout: "null",
|
||||||
|
stderr: "inherit",
|
||||||
|
}).spawn();
|
||||||
|
const origin = `http://127.0.0.1:${port}`;
|
||||||
|
await Deno.mkdir(artifact, { recursive: true });
|
||||||
|
const browser = await chromium.launch();
|
||||||
|
try {
|
||||||
|
await waitForServer(`${origin}/health`);
|
||||||
|
for (const theme of ["light", "dark"] as const) {
|
||||||
|
for (const width of [1440, 768, 390]) {
|
||||||
|
const context = await browser.newContext({
|
||||||
|
viewport: { width, height: 900 },
|
||||||
|
colorScheme: theme,
|
||||||
|
});
|
||||||
|
const page = await context.newPage();
|
||||||
|
const pageErrors: string[] = [];
|
||||||
|
page.on("pageerror", (error) => pageErrors.push(error.message));
|
||||||
|
const requests: unknown[] = [];
|
||||||
|
let release!: () => void;
|
||||||
|
const held = new Promise<void>((resolve) => {
|
||||||
|
release = resolve;
|
||||||
|
});
|
||||||
|
await page.route(
|
||||||
|
"**/subjektiv/subjects/pending-subject",
|
||||||
|
async (route) => {
|
||||||
|
if (route.request().method() !== "DELETE") {
|
||||||
|
return route.continue();
|
||||||
|
}
|
||||||
|
requests.push(route.request().postDataJSON());
|
||||||
|
if (requests.length === 1) await held;
|
||||||
|
await route.continue();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
|
||||||
|
await page.locator(".subject-delete-open").waitFor();
|
||||||
|
await page.screenshot({
|
||||||
|
path: join(artifact, `${theme}-${width}-initial.png`),
|
||||||
|
});
|
||||||
|
await page.locator(".subject-delete-open").click();
|
||||||
|
assert(
|
||||||
|
await page.getByText("This is irreversible.", { exact: true })
|
||||||
|
.isVisible(),
|
||||||
|
);
|
||||||
|
await assertNoPageWideOverflow(page);
|
||||||
|
await page.screenshot({
|
||||||
|
path: join(artifact, `${theme}-${width}-confirmation.png`),
|
||||||
|
});
|
||||||
|
await page.getByRole("button", { name: "Cancel", exact: true })
|
||||||
|
.click();
|
||||||
|
assertEquals(requests.length, 0);
|
||||||
|
await confirm(page);
|
||||||
|
await page.locator('[data-delete-state="running"]').waitFor();
|
||||||
|
assertEquals(await page.locator(".subject-overview").count(), 0);
|
||||||
|
await page.screenshot({
|
||||||
|
path: join(artifact, `${theme}-${width}-running.png`),
|
||||||
|
});
|
||||||
|
release();
|
||||||
|
await page.locator('[data-delete-state="pending"]').waitFor();
|
||||||
|
assertEquals(requests.length, 1);
|
||||||
|
const saved = await page.evaluate(() =>
|
||||||
|
Object.keys(sessionStorage).filter((key) =>
|
||||||
|
key.startsWith("yoi.subject-delete:")
|
||||||
|
).map((key) => JSON.parse(sessionStorage.getItem(key)!))
|
||||||
|
);
|
||||||
|
assertEquals(saved, [{
|
||||||
|
subjectId: "pending-subject",
|
||||||
|
operationId: (requests[0] as { operation_id: string }).operation_id,
|
||||||
|
}]);
|
||||||
|
await page.screenshot({
|
||||||
|
path: join(artifact, `${theme}-${width}-pending.png`),
|
||||||
|
});
|
||||||
|
await page.reload();
|
||||||
|
await page.locator('[data-delete-state="unknown"]').waitFor();
|
||||||
|
assertEquals(await page.locator(".subject-overview").count(), 0);
|
||||||
|
await page.locator(".subject-delete-retry").click();
|
||||||
|
await page.locator('[data-delete-state="completed"]').waitFor();
|
||||||
|
assertEquals(requests.length, 2);
|
||||||
|
assertEquals(requests[0], requests[1]);
|
||||||
|
assertEquals(
|
||||||
|
await page.evaluate(() =>
|
||||||
|
Object.keys(sessionStorage).filter((key) =>
|
||||||
|
key.startsWith("yoi.subject-delete:")
|
||||||
|
)
|
||||||
|
),
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
await page.screenshot({
|
||||||
|
path: join(artifact, `${theme}-${width}-completed.png`),
|
||||||
|
});
|
||||||
|
await assertNoPageWideOverflow(page);
|
||||||
|
assertEquals(pageErrors, []);
|
||||||
|
await context.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const subject of ["unknown-subject", "legacy-subject"]) {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
const requests: unknown[] = [];
|
||||||
|
page.on("request", (request) => {
|
||||||
|
if (request.method() === "DELETE") {
|
||||||
|
requests.push(request.postDataJSON());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await page.goto(`${origin}/w/memory-review/memory/${subject}`);
|
||||||
|
await page.locator(".subject-delete-open").waitFor();
|
||||||
|
if (subject === "legacy-subject") {
|
||||||
|
assert(
|
||||||
|
await page.getByText("Legacy ended Subject", { exact: true })
|
||||||
|
.isVisible(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await confirm(page);
|
||||||
|
await page.locator(
|
||||||
|
`[data-delete-state="${
|
||||||
|
subject === "unknown-subject" ? "unknown" : "pending"
|
||||||
|
}"]`,
|
||||||
|
).waitFor();
|
||||||
|
await page.locator(".subject-delete-retry").click();
|
||||||
|
await page.locator('[data-delete-state="completed"]').waitFor();
|
||||||
|
assertEquals(requests.length, 2);
|
||||||
|
assertEquals(requests[0], requests[1]);
|
||||||
|
await page.close();
|
||||||
|
}
|
||||||
|
// 4xxでもkeyを捨てない。権限の変化を新しい依頼へ置換しない。
|
||||||
|
{
|
||||||
|
const page = await browser.newPage();
|
||||||
|
const requests: unknown[] = [];
|
||||||
|
await page.route(
|
||||||
|
"**/subjektiv/subjects/pending-subject",
|
||||||
|
async (route) => {
|
||||||
|
if (route.request().method() !== "DELETE") return route.continue();
|
||||||
|
requests.push(route.request().postDataJSON());
|
||||||
|
if (requests.length === 1) {
|
||||||
|
await route.fulfill({
|
||||||
|
status: 403,
|
||||||
|
json: { error: "forbidden", message: "Permission changed" },
|
||||||
|
});
|
||||||
|
} else {await route.fulfill({
|
||||||
|
json: {
|
||||||
|
subject_id: "pending-subject",
|
||||||
|
operation_id: (requests[0] as any).operation_id,
|
||||||
|
completed: true,
|
||||||
|
},
|
||||||
|
});}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
|
||||||
|
await confirm(page);
|
||||||
|
await page.locator('[data-delete-state="failed"]').waitFor();
|
||||||
|
await page.screenshot({ path: join(artifact, "rejected.png") });
|
||||||
|
await page.locator(".subject-delete-retry").click();
|
||||||
|
await page.locator('[data-delete-state="completed"]').waitFor();
|
||||||
|
assertEquals(requests[0], requests[1]);
|
||||||
|
await page.close();
|
||||||
|
}
|
||||||
|
// 同一SPA内で他対象へ移動してから旧応答が届いても、新対象を消さない。
|
||||||
|
{
|
||||||
|
const page = await browser.newPage();
|
||||||
|
let release!: () => void;
|
||||||
|
const held = new Promise<void>((resolve) => {
|
||||||
|
release = resolve;
|
||||||
|
});
|
||||||
|
await page.route(
|
||||||
|
"**/subjektiv/subjects/pending-subject",
|
||||||
|
async (route) => {
|
||||||
|
if (route.request().method() !== "DELETE") return route.continue();
|
||||||
|
const body = route.request().postDataJSON();
|
||||||
|
await held;
|
||||||
|
await route.fulfill({
|
||||||
|
json: {
|
||||||
|
subject_id: "pending-subject",
|
||||||
|
operation_id: body.operation_id,
|
||||||
|
completed: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
|
||||||
|
await confirm(page);
|
||||||
|
await page.locator('[data-delete-state="running"]').waitFor();
|
||||||
|
await page.getByRole("link", { name: "← All subjects", exact: true })
|
||||||
|
.click();
|
||||||
|
await page.locator(".subject-list a").first().click();
|
||||||
|
await page.locator(".subject-overview").waitFor();
|
||||||
|
const target = page.url();
|
||||||
|
const response = page.waitForResponse((r) =>
|
||||||
|
r.request().method() === "DELETE"
|
||||||
|
);
|
||||||
|
release();
|
||||||
|
await response;
|
||||||
|
assertEquals(page.url(), target);
|
||||||
|
assertEquals(
|
||||||
|
await page.locator('[data-delete-state="completed"]').count(),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
assertEquals(await page.locator(".subject-overview").count(), 1);
|
||||||
|
await page.close();
|
||||||
|
}
|
||||||
|
{
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.goto(
|
||||||
|
`${origin}/w/memory-review/memory/legacy-subject?persona=reader`,
|
||||||
|
);
|
||||||
|
await page.locator(".subject-overview").waitFor();
|
||||||
|
assertEquals(await page.locator(".subject-deletion").count(), 0);
|
||||||
|
await page.screenshot({ path: join(artifact, "non-owner.png") });
|
||||||
|
await page.close();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await browser.close();
|
||||||
|
server.kill("SIGTERM");
|
||||||
|
await server.status;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
import {
|
||||||
|
deleteSubjektivSubject,
|
||||||
|
parseSubjektivSubjectDeleteResponse,
|
||||||
|
SubjektivSubjectDeleteError,
|
||||||
|
} from "../src/lib/workspace/memory/api.ts";
|
||||||
|
import {
|
||||||
|
clearSubjectDeleteAttempt,
|
||||||
|
readSubjectDeleteAttempt,
|
||||||
|
saveSubjectDeleteAttempt,
|
||||||
|
subjectDeleteStorageKey,
|
||||||
|
} from "../src/lib/workspace/memory/subject-delete.ts";
|
||||||
|
|
||||||
|
declare const Deno: {
|
||||||
|
test(name: string, fn: () => void | Promise<void>): void;
|
||||||
|
};
|
||||||
|
|
||||||
|
function equal(actual: unknown, expected: unknown) {
|
||||||
|
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
||||||
|
throw new Error(
|
||||||
|
`Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Deno.test("Subject deletion uses DELETE with the same request after fence-only success", async () => {
|
||||||
|
const calls: unknown[] = [];
|
||||||
|
const fetcher = ((input, init) => {
|
||||||
|
calls.push([input, init]);
|
||||||
|
return Promise.resolve(
|
||||||
|
Response.json({
|
||||||
|
subject_id: "s/1",
|
||||||
|
operation_id: "same-operation",
|
||||||
|
completed: calls.length > 1,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}) as typeof fetch;
|
||||||
|
for (const completed of [false, true]) {
|
||||||
|
equal(
|
||||||
|
await deleteSubjektivSubject(fetcher, "w/1", "s/1", {
|
||||||
|
operation_id: "same-operation",
|
||||||
|
}),
|
||||||
|
{ subject_id: "s/1", operation_id: "same-operation", completed },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
equal(calls[0], calls[1]);
|
||||||
|
equal(calls[0], ["/api/w/w%2F1/subjektiv/subjects/s%2F1", {
|
||||||
|
method: "DELETE",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: '{"operation_id":"same-operation"}',
|
||||||
|
}]);
|
||||||
|
});
|
||||||
|
Deno.test("Subject deletion never confirms a malformed or wrong-target response", () => {
|
||||||
|
for (
|
||||||
|
const payload of [
|
||||||
|
{ subject_id: "other", operation_id: "op", completed: true },
|
||||||
|
{ subject_id: "s", operation_id: "other", completed: true },
|
||||||
|
{ subject_id: "s", operation_id: "op", completed: "true" },
|
||||||
|
{ subject_id: "s", operation_id: "op" },
|
||||||
|
{ subject_id: "s", operation_id: "op", completed: true, extra: 1 },
|
||||||
|
]
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
parseSubjektivSubjectDeleteResponse(payload, "s", "op");
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new Error("Accepted an unconfirmed response");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Deno.test("Subject deletion classifies rejection separately from unknown transport/server/response outcomes", async () => {
|
||||||
|
for (
|
||||||
|
const item of [
|
||||||
|
{
|
||||||
|
fetcher: () => Promise.reject(new Error("lost")),
|
||||||
|
kind: "unknown_outcome",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fetcher: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
Response.json({ error: "forbidden" }, { status: 403 }),
|
||||||
|
),
|
||||||
|
kind: "rejected",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fetcher: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
Response.json({ error: "conflict" }, { status: 409 }),
|
||||||
|
),
|
||||||
|
kind: "rejected",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fetcher: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
Response.json({ error: "uncertain" }, { status: 500 }),
|
||||||
|
),
|
||||||
|
kind: "unknown_outcome",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fetcher: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
Response.json({
|
||||||
|
subject_id: "wrong",
|
||||||
|
operation_id: "op",
|
||||||
|
completed: true,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
kind: "unknown_outcome",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
await deleteSubjektivSubject(item.fetcher as typeof fetch, "w", "s", {
|
||||||
|
operation_id: "op",
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof SubjektivSubjectDeleteError)) throw error;
|
||||||
|
equal(error.kind, item.kind);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new Error("Unexpected success");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Deno.test("Retry input survives reload independently of Subject GET and is isolated by Workspace/Subject", () => {
|
||||||
|
const values = new Map<string, string>();
|
||||||
|
const storage = {
|
||||||
|
getItem: (key: string) => values.get(key) ?? null,
|
||||||
|
setItem: (key: string, value: string) => {
|
||||||
|
values.set(key, value);
|
||||||
|
},
|
||||||
|
removeItem: (key: string) => {
|
||||||
|
values.delete(key);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const attempt = {
|
||||||
|
subjectId: "s",
|
||||||
|
operationId: "same-op",
|
||||||
|
};
|
||||||
|
saveSubjectDeleteAttempt(storage, "w", attempt);
|
||||||
|
equal(readSubjectDeleteAttempt(storage, "w", "s"), attempt);
|
||||||
|
equal(readSubjectDeleteAttempt(storage, "other", "s"), null);
|
||||||
|
equal(readSubjectDeleteAttempt(storage, "w", "other"), null);
|
||||||
|
saveSubjectDeleteAttempt(storage, "other", attempt);
|
||||||
|
clearSubjectDeleteAttempt(storage, "w", "s");
|
||||||
|
equal(readSubjectDeleteAttempt(storage, "w", "s"), null);
|
||||||
|
equal(readSubjectDeleteAttempt(storage, "other", "s"), attempt);
|
||||||
|
values.set(subjectDeleteStorageKey("w", "s"), '{"subjectId":"other"}');
|
||||||
|
try {
|
||||||
|
readSubjectDeleteAttempt(storage, "w", "s");
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw new Error("Corrupt retry input allowed new deletion");
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user