fix: recover unpublished Subject creation and preserve auth rejections

This commit is contained in:
2026-10-19 18:56:24 +09:00
parent c961f244ab
commit 8e40fae6f4
20 changed files with 2243 additions and 58 deletions
+55
View File
@@ -1140,8 +1140,36 @@ pub struct SubjectReleaseResponse {
pub state: SubjectReleaseState, pub state: SubjectReleaseState,
} }
/// Irreversibly fence an unpublished failed Subject create. Exact retries must retain
/// all four fields; this is distinct from releasing an existing Subject Controller.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SubjectCreateRecoveryRequest {
pub workspace_id: String,
pub worker_id: WorkerId,
pub create_fingerprint: String,
pub operation_id: String,
}
/// Durable admission-fence proof plus freshly confirmed execution absence.
/// An error (including a lost response) is never evidence of absence.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SubjectCreateRecoveryResponse {
pub workspace_id: String,
pub worker_id: WorkerId,
pub create_fingerprint: String,
pub operation_id: String,
}
#[api(reqwest, axum)] #[api(reqwest, axum)]
pub trait RuntimeApi { pub trait RuntimeApi {
#[post("/v1/subject-create-recovery", status = 200, error_status = 400)]
async fn recover_subject_create(
&self,
#[body] request: SubjectCreateRecoveryRequest,
) -> Result<SubjectCreateRecoveryResponse, RuntimeApiError>;
#[post("/v1/subject-release", status = 200, error_status = 400)] #[post("/v1/subject-release", status = 200, error_status = 400)]
async fn subject_release( async fn subject_release(
&self, &self,
@@ -1675,6 +1703,18 @@ mod tests {
) -> Result<SessionMaterialPinResponse, RuntimeApiError> { ) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
Err(test_error(501)) Err(test_error(501))
} }
async fn recover_subject_create(
&self,
request: SubjectCreateRecoveryRequest,
) -> Result<SubjectCreateRecoveryResponse, RuntimeApiError> {
Ok(SubjectCreateRecoveryResponse {
workspace_id: request.workspace_id,
worker_id: request.worker_id,
create_fingerprint: request.create_fingerprint,
operation_id: request.operation_id,
})
}
async fn subject_release( async fn subject_release(
&self, &self,
_request: SubjectReleaseRequest, _request: SubjectReleaseRequest,
@@ -1820,6 +1860,21 @@ mod tests {
.build() .build()
.unwrap(); .unwrap();
let recovery = SubjectCreateRecoveryRequest {
workspace_id: "workspace-a".into(),
worker_id: WorkerId::now_v7(),
create_fingerprint: "failed-create-fingerprint".into(),
operation_id: "recovery-1".into(),
};
let response = client
.recover_subject_create(recovery.clone())
.await
.unwrap();
assert_eq!(response.workspace_id, recovery.workspace_id);
assert_eq!(response.worker_id, recovery.worker_id);
assert_eq!(response.create_fingerprint, recovery.create_fingerprint);
assert_eq!(response.operation_id, recovery.operation_id);
let summary = client.runtime_summary().await.unwrap(); let summary = client.runtime_summary().await.unwrap();
assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip")); assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip"));
let workers = client let workers = client
+100
View File
@@ -67,6 +67,27 @@ impl SubjektivStore {
}) })
} }
/// Absence proof for failed body creation, including bindings never attached to a Session
/// and released lifetimes. The Host must hold its lifecycle fence across this observation
/// and recovery; absence alone does not fence a delayed Runtime create.
pub fn journal_body_is_unpublished(
&self,
subject: &str,
runtime: &str,
worker: &str,
) -> JournalResult<bool> {
self.database.try_with_connection(|tx| {
Ok(tx.query_row(
"SELECT NOT EXISTS(SELECT 1 FROM journal_writer_bindings
WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)
AND NOT EXISTS(SELECT 1 FROM subject_session_attributions
WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)",
params![subject, runtime, worker],
|row| row.get(0),
)?)
})
}
pub fn journal_worker_was_released( pub fn journal_worker_was_released(
&self, &self,
subject: &str, subject: &str,
@@ -203,3 +224,82 @@ impl SubjektivStore {
}) })
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unpublished_body_rejects_orphan_and_released_bindings_and_attribution_only() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _workspace, store) = crate::tests::open_store(temp.path(), "absence-tests");
let subject = store
.create_subject(crate::SubjectRole::new("companion").unwrap())
.unwrap();
let other_subject = store
.create_subject(crate::SubjectRole::new("companion").unwrap())
.unwrap();
assert!(
store
.journal_body_is_unpublished(&subject.id, "runtime", "worker")
.unwrap()
);
// No Session was attached: an orphan binding is still published body evidence.
let binding = store
.claim_journal_writer(&subject.id, "runtime", "worker")
.unwrap();
assert!(
!store
.journal_body_is_unpublished(&subject.id, "runtime", "worker")
.unwrap()
);
for (subject_id, runtime, worker) in [
(other_subject.id.as_str(), "runtime", "worker"),
(subject.id.as_str(), "other-runtime", "worker"),
(subject.id.as_str(), "runtime", "other-worker"),
] {
assert!(
store
.journal_body_is_unpublished(subject_id, runtime, worker)
.unwrap()
);
}
store.release_journal_writer(&binding).unwrap();
assert!(
!store
.journal_body_is_unpublished(&subject.id, "runtime", "worker")
.unwrap()
);
store
.record_session_attribution(
SubjectSessionAttribution::new(
&subject.id,
"runtime",
"attribution-only",
"session",
)
.unwrap(),
)
.unwrap();
assert!(
!store
.journal_body_is_unpublished(&subject.id, "runtime", "attribution-only")
.unwrap()
);
assert!(
store
.journal_body_is_unpublished(&subject.id, "other-runtime", "attribution-only")
.unwrap()
);
assert!(
store
.journal_body_is_unpublished(&other_subject.id, "runtime", "attribution-only")
.unwrap()
);
assert!(
store
.journal_body_is_unpublished(&subject.id, "runtime", "never-published")
.unwrap()
);
}
}
+16
View File
@@ -567,6 +567,15 @@ impl fmt::Debug for WorkerProtocolTransport {
pub trait WorkerExecutionBackend: Send + Sync + 'static { pub trait WorkerExecutionBackend: Send + Sync + 'static {
fn backend_id(&self) -> &str; fn backend_id(&self) -> &str;
/// Confirm actual absence under the backend Worker operation lock. Runtime has
/// already durably fenced admission; unsupported or uncertain backends must fail.
fn recover_subject_create(
&self,
_request: &runtime_api::SubjectCreateRecoveryRequest,
) -> Result<(), String> {
Err("execution backend cannot confirm Subject create execution absence".into())
}
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot. /// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
fn subject_release( fn subject_release(
&self, &self,
@@ -839,6 +848,13 @@ impl WorkerExecutionBackendRef {
self.backend.fetch_workspace_config(request) self.backend.fetch_workspace_config(request)
} }
pub(crate) fn recover_subject_create(
&self,
request: &runtime_api::SubjectCreateRecoveryRequest,
) -> Result<(), String> {
self.backend.recover_subject_create(request)
}
pub(crate) fn subject_release( pub(crate) fn subject_release(
&self, &self,
request: &runtime_api::SubjectReleaseRequest, request: &runtime_api::SubjectReleaseRequest,
+291 -35
View File
@@ -39,9 +39,10 @@ use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
const SCHEMA_VERSION: u32 = 11; const SCHEMA_VERSION: u32 = 12;
const EXECUTION_SCHEMA_VERSION: u32 = 10;
// Frozen source schemas: do not move their meaning when adding a migration. // Frozen source schemas: do not move their meaning when adding a migration.
const SOURCE_SCHEMA_VERSION: u32 = 11;
const EXECUTION_SCHEMA_VERSION: u32 = 10;
const AGGREGATE_SCHEMA_VERSION: u32 = 9; const AGGREGATE_SCHEMA_VERSION: u32 = 9;
const PREVIOUS_SCHEMA_VERSION: u32 = 8; const PREVIOUS_SCHEMA_VERSION: u32 = 8;
const LEGACY_SCHEMA_VERSION: u32 = 7; const LEGACY_SCHEMA_VERSION: u32 = 7;
@@ -480,6 +481,8 @@ pub(crate) struct OpenedFsRuntimeStore {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) struct PersistedRuntimeState { pub(crate) struct PersistedRuntimeState {
pub(crate) subject_create_recovery_fences:
BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
pub(crate) subject_releases: Vec<( pub(crate) subject_releases: Vec<(
runtime_api::SubjectReleaseRequest, runtime_api::SubjectReleaseRequest,
runtime_api::SubjectReleaseState, runtime_api::SubjectReleaseState,
@@ -641,8 +644,8 @@ fn plan_runtime_store_migration(
format!("Runtime store schema version {schema_version} is out of range"), format!("Runtime store schema version {schema_version} is out of range"),
) )
})?; })?;
let staging = migration_sibling(root, "schema-v11-staging")?; let staging = migration_sibling(root, "schema-v12-staging")?;
let backup = migration_sibling(root, "pre-schema-v11-backup")?; let backup = migration_sibling(root, "pre-schema-v12-backup")?;
if staging.exists() || backup.exists() { if staging.exists() || backup.exists() {
return Err(runtime_store_corrupt( return Err(runtime_store_corrupt(
root, root,
@@ -676,11 +679,12 @@ fn plan_runtime_store_migration(
| PREVIOUS_SCHEMA_VERSION | PREVIOUS_SCHEMA_VERSION
| AGGREGATE_SCHEMA_VERSION | AGGREGATE_SCHEMA_VERSION
| EXECUTION_SCHEMA_VERSION | EXECUTION_SCHEMA_VERSION
| SOURCE_SCHEMA_VERSION
) { ) {
return Err(runtime_store_corrupt( return Err(runtime_store_corrupt(
&runtime_path, &runtime_path,
format!( format!(
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
), ),
)); ));
} }
@@ -1010,11 +1014,16 @@ fn migrate_worker_record(
mapping: Option<&LegacyWorkerIdentityMapping>, mapping: Option<&LegacyWorkerIdentityMapping>,
identity_path: &Path, identity_path: &Path,
) -> Result<WorkerAggregateRecord, RuntimeError> { ) -> Result<WorkerAggregateRecord, RuntimeError> {
if source_schema_version == EXECUTION_SCHEMA_VERSION { if matches!(
source_schema_version,
EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION
) {
// Schemas 10 and 11 already use the current Worker aggregate layout.
// Only advance its version; never regenerate Restore admission identity.
if identity_document if identity_document
.get("schema_version") .get("schema_version")
.and_then(serde_json::Value::as_u64) .and_then(serde_json::Value::as_u64)
!= Some(u64::from(EXECUTION_SCHEMA_VERSION)) != Some(u64::from(source_schema_version))
{ {
return Err(runtime_store_corrupt( return Err(runtime_store_corrupt(
identity_path, identity_path,
@@ -1405,11 +1414,18 @@ fn migrate_runtime_document(
"schema_version".to_string(), "schema_version".to_string(),
serde_json::Value::from(SCHEMA_VERSION), serde_json::Value::from(SCHEMA_VERSION),
); );
object.remove("workers"); // Intermediate schema-11 writers may already have saved irreversible fences.
// This legacy cache is not Runtime lifecycle or restore authority. Bundles // Default only a missing field; malformed or existing fences must not be erased.
// are reacquired from Workspace content authority, never renumbered/rehash-pinned here. object
object.remove("config_bundles"); .entry("subject_create_recovery_fences")
object.remove("next_worker_sequence"); .or_insert_with(|| serde_json::json!({}));
if source_schema_version != SOURCE_SCHEMA_VERSION {
object.remove("workers");
// This legacy cache is not Runtime lifecycle or restore authority. Bundles
// are reacquired from Workspace content authority, never renumbered/rehash-pinned here.
object.remove("config_bundles");
object.remove("next_worker_sequence");
}
let snapshot: RuntimeSnapshot = serde_json::from_value(document.clone()).map_err(|error| { let snapshot: RuntimeSnapshot = serde_json::from_value(document.clone()).map_err(|error| {
runtime_store_corrupt( runtime_store_corrupt(
runtime_path, runtime_path,
@@ -1523,14 +1539,16 @@ fn recover_runtime_store_migration(root: &Path) -> Result<(), RuntimeError> {
"pre-schema-v9-backup", "pre-schema-v9-backup",
"schema-v10-staging", "schema-v10-staging",
"pre-schema-v10-backup", "pre-schema-v10-backup",
"schema-v11-staging",
"pre-schema-v11-backup",
] { ] {
let earlier = migration_sibling(root, suffix)?; let earlier = migration_sibling(root, suffix)?;
if earlier.exists() { if earlier.exists() {
return Err(runtime_store_corrupt(root, "unfinished earlier Runtime migration requires explicit recovery before schema-v11 startup".into())); return Err(runtime_store_corrupt(root, "unfinished earlier Runtime migration requires explicit recovery before schema-v12 startup".into()));
} }
} }
let staging = migration_sibling(root, "schema-v11-staging")?; let staging = migration_sibling(root, "schema-v12-staging")?;
let backup = migration_sibling(root, "pre-schema-v11-backup")?; let backup = migration_sibling(root, "pre-schema-v12-backup")?;
let root_exists = migration_directory_exists(root)?; let root_exists = migration_directory_exists(root)?;
let staging_exists = migration_directory_exists(&staging)?; let staging_exists = migration_directory_exists(&staging)?;
let backup_exists = migration_directory_exists(&backup)?; let backup_exists = migration_directory_exists(&backup)?;
@@ -1732,8 +1750,8 @@ fn migrate_runtime_store(
if !plan.migration_required { if !plan.migration_required {
return Ok(plan); return Ok(plan);
} }
let staging = migration_sibling(root, "schema-v11-staging")?; let staging = migration_sibling(root, "schema-v12-staging")?;
let backup = migration_sibling(root, "pre-schema-v11-backup")?; let backup = migration_sibling(root, "pre-schema-v12-backup")?;
if staging.exists() || backup.exists() { if staging.exists() || backup.exists() {
return Err(runtime_store_corrupt( return Err(runtime_store_corrupt(
root, root,
@@ -1908,7 +1926,10 @@ fn migrate_runtime_store_in_place(
)); ));
} }
} }
if let Some(metadata) = metadata { // Schema 11 metadata was validated above but needs no conversion.
if let Some(metadata) = metadata
&& plan.current_schema_version != SOURCE_SCHEMA_VERSION
{
atomic_write_json( atomic_write_json(
&migrated_dir.join(WORKER_METADATA_FILE), &migrated_dir.join(WORKER_METADATA_FILE),
&metadata, &metadata,
@@ -1927,7 +1948,11 @@ fn migrate_runtime_store_in_place(
), ),
)); ));
} }
crate::retention::migrate_staged_sessions(root)?; // Schema 11 already migrated Sessions; the recovery-fence bump must not
// rewrite retained history or archive attestations.
if plan.current_schema_version != SOURCE_SCHEMA_VERSION {
crate::retention::migrate_staged_sessions(root)?;
}
let (document, _) = migrate_runtime_document( let (document, _) = migrate_runtime_document(
document, document,
plan.current_schema_version, plan.current_schema_version,
@@ -1944,6 +1969,8 @@ fn migrate_runtime_store_in_place(
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
struct RuntimeSnapshot { struct RuntimeSnapshot {
#[serde(default)]
subject_create_recovery_fences: BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
#[serde(default)] #[serde(default)]
subject_releases: Vec<( subject_releases: Vec<(
runtime_api::SubjectReleaseRequest, runtime_api::SubjectReleaseRequest,
@@ -1980,6 +2007,7 @@ fn record_worker_load_diagnostic(
impl RuntimeSnapshot { impl RuntimeSnapshot {
fn from_persisted(state: &PersistedRuntimeState) -> Self { fn from_persisted(state: &PersistedRuntimeState) -> Self {
Self { Self {
subject_create_recovery_fences: state.subject_create_recovery_fences.clone(),
subject_releases: state.subject_releases.clone(), subject_releases: state.subject_releases.clone(),
schema_version: SCHEMA_VERSION, schema_version: SCHEMA_VERSION,
display_name: state.display_name.clone(), display_name: state.display_name.clone(),
@@ -2003,6 +2031,33 @@ impl RuntimeSnapshot {
), ),
}); });
} }
for (worker_id, fence) in &self.subject_create_recovery_fences {
let request = &fence.request;
let response_matches = fence.response.as_ref().is_none_or(|response| {
response.worker_id == request.worker_id
&& response.workspace_id == request.workspace_id
&& response.create_fingerprint == request.create_fingerprint
&& response.operation_id == request.operation_id
});
if *worker_id != request.worker_id
|| request.workspace_id.trim().is_empty()
|| request.create_fingerprint.trim().is_empty()
|| request.create_fingerprint.len() > 512
|| request.operation_id.trim().is_empty()
|| request.operation_id.len() > 512
|| self
.workspace_owners
.get(&request.workspace_id)
.is_none_or(|owner| owner.trim().is_empty())
|| !response_matches
{
return Err(RuntimeError::StoreCorrupt {
operation: "read Subject create recovery fence",
path: path.to_path_buf(),
message: "invalid recovery identity, receipt or Workspace owner".into(),
});
}
}
if self.backend != RuntimeBackendKind::FsStore { if self.backend != RuntimeBackendKind::FsStore {
return Err(RuntimeError::StoreCorrupt { return Err(RuntimeError::StoreCorrupt {
operation: "read runtime snapshot", operation: "read runtime snapshot",
@@ -2018,6 +2073,7 @@ impl RuntimeSnapshot {
workers: BTreeMap<WorkerId, PersistedWorkerRecord>, workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
) -> PersistedRuntimeState { ) -> PersistedRuntimeState {
PersistedRuntimeState { PersistedRuntimeState {
subject_create_recovery_fences: self.subject_create_recovery_fences,
subject_releases: self.subject_releases, subject_releases: self.subject_releases,
display_name: self.display_name, display_name: self.display_name,
status: self.status, status: self.status,
@@ -2708,6 +2764,199 @@ mod tests {
document["schema_version"].as_u64().unwrap() document["schema_version"].as_u64().unwrap()
} }
#[test]
fn schema_v11_migration_preserves_identity_history_and_recovery_fences_on_reopen() {
// Cover the released schema (no field) and intermediate writers with
// either a pending fence or a completed recovery receipt.
for fence_state in ["absent", "pending", "completed"] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let worker_id = WorkerId::now_v7();
let fenced_worker_id = WorkerId::now_v7();
let worker_dir = root.join(WORKERS_DIR).join(worker_id.to_string());
fs::create_dir_all(worker_dir.join("session")).unwrap();
let runtime_path = root.join(RUNTIME_FILE);
let worker_path = worker_dir.join(WORKER_FILE);
let metadata_path = worker_dir.join(WORKER_METADATA_FILE);
let history_path = worker_dir.join("session/history.json");
let history = b"[{\"role\":\"user\",\"content\":\"retained history\"}]\n";
fs::write(&history_path, history).unwrap();
let metadata = serde_json::to_vec(&serde_json::json!({
"worker_name": runtime_worker_name(worker_id),
"workspace_id": "workspace-a",
"retained_extension": {"history": "must not be rewritten"}
}))
.unwrap();
fs::write(&metadata_path, &metadata).unwrap();
let mut runtime = serde_json::json!({
"schema_version": 11,
"display_name": "schema-11 Runtime",
"backend": "fs_store",
"status": "running",
"next_diagnostic_id": 2,
"config_bundles": {},
"workspace_owners": {"workspace-a": "owner-a"},
"subject_releases": [],
"diagnostics": [{
"id": 1, "worker_ref": {"worker_id": worker_id},
"severity": "warning", "code": "retained_history",
"message": "preserve diagnostic identity"
}]
});
let recovery_identity = serde_json::json!({
"workspace_id": "workspace-a", "worker_id": fenced_worker_id,
"create_fingerprint": "failed-create", "operation_id": "recover-a"
});
if fence_state != "absent" {
runtime["subject_create_recovery_fences"] = serde_json::json!({
fenced_worker_id.to_string(): {
"request": recovery_identity,
"response": if fence_state == "completed" {
recovery_identity.clone()
} else {
serde_json::Value::Null
}
}
});
}
atomic_write_json(&runtime_path, &runtime, "write schema-11 fixture").unwrap();
let runtime_before = fs::read(&runtime_path).unwrap();
let mut request = schema_v7_worker_document(worker_id)["request"].clone();
request["workspace_api"] = serde_json::json!({
"workspace_id": "workspace-a", "base_url": "https://workspace.invalid"
});
// Serialize defaults as a schema-11 writer would, without running
// a migration or generating a fresh Restore guard for the fixture.
let request: CreateWorkerRequest = serde_json::from_value(request).unwrap();
let worker: WorkerAggregateRecord = serde_json::from_value(serde_json::json!({
"schema_version": 11,
"restore_guard": {
"observation_token": "retained-restore-token",
"active_request_id": null, "owners": {}
},
"worker_ref": {"worker_id": worker_id}, "worker_id": worker_id,
"profile": request.profile,
"display_name": null,
"profile_source": request.profile_source.reference(),
"config_bundle": null,
"created_at_ms": 1234,
"status": "running",
"last_finished_submission_request_id": "finished-submission",
"workspace_id": "workspace-a",
"execution_state": {"state": "available", "execution": {
"request": request, "binding": {}, "restore_intent": "automatic"
}}
}))
.unwrap();
let mut expected_worker = serde_json::to_value(&worker).unwrap();
atomic_write_json(&worker_path, &worker, "write schema-11 Worker").unwrap();
let worker_before = fs::read(&worker_path).unwrap();
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-test");
let plan = FsRuntimeStore::migration_plan(&options).unwrap();
assert_eq!(plan.current_schema_version, 11);
assert_eq!(plan.target_schema_version, 12);
assert!(plan.migration_required);
assert_eq!(plan.worker_count, 1);
assert_eq!(plan.migrated_worker_aggregate_count, 0);
assert!(plan.mappings.is_empty());
assert_eq!(fs::read(&runtime_path).unwrap(), runtime_before);
assert_eq!(fs::read(&worker_path).unwrap(), worker_before);
// The direct reader still rejects a previous schema: migration is
// explicit, not permissive deserialization of a newer format.
let snapshot: RuntimeSnapshot = serde_json::from_value(runtime.clone()).unwrap();
assert!(snapshot.validate(&runtime_path).is_err());
let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap();
let state = opened.state.as_ref().unwrap();
let persisted_worker = state.workers.get(&worker_id).unwrap();
expected_worker["schema_version"] = serde_json::json!(12);
assert_eq!(
serde_json::to_value(WorkerAggregateRecord::from_persisted(persisted_worker))
.unwrap(),
expected_worker
);
runtime["schema_version"] = serde_json::json!(12);
if fence_state == "absent" {
runtime["subject_create_recovery_fences"] = serde_json::json!({});
}
assert_eq!(
serde_json::to_value(&state.subject_create_recovery_fences).unwrap(),
runtime["subject_create_recovery_fences"]
);
assert_eq!(
read_json::<serde_json::Value>(&runtime_path, "read migrated Runtime").unwrap(),
runtime
);
assert_eq!(
read_json::<serde_json::Value>(&worker_path, "read migrated Worker").unwrap(),
expected_worker
);
// Exercise the normal writer too: it must retain intermediate fences.
opened.store.write_runtime_snapshot(state).unwrap();
drop(opened);
let runtime_once = fs::read(&runtime_path).unwrap();
let worker_once = fs::read(&worker_path).unwrap();
let reopened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap();
assert_eq!(
serde_json::to_value(
&reopened
.state
.as_ref()
.unwrap()
.subject_create_recovery_fences
)
.unwrap(),
runtime["subject_create_recovery_fences"]
);
drop(reopened);
let plan = FsRuntimeStore::migrate(&options).unwrap();
assert!(!plan.migration_required);
assert_eq!(plan.current_schema_version, 12);
assert_eq!(plan.target_schema_version, 12);
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
assert_eq!(fs::read(&runtime_path).unwrap(), runtime_once);
assert_eq!(fs::read(&worker_path).unwrap(), worker_once);
assert_eq!(fs::read(&metadata_path).unwrap(), metadata);
assert_eq!(fs::read(&history_path).unwrap(), history);
assert!(
!migration_sibling(&root, "schema-v12-staging")
.unwrap()
.exists()
);
assert!(
!migration_sibling(&root, "pre-schema-v12-backup")
.unwrap()
.exists()
);
}
}
#[test]
fn schema_v11_migration_rejects_invalid_fences_without_replacing_them() {
for fences in [
serde_json::Value::Null,
serde_json::json!({"invalid-worker": {}}),
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
write_empty_schema_v8_store(&root);
let path = root.join(RUNTIME_FILE);
let mut runtime: serde_json::Value = read_json(&path, "read fixture").unwrap();
runtime["schema_version"] = serde_json::json!(11);
runtime["subject_create_recovery_fences"] = fences;
atomic_write_json(&path, &runtime, "write invalid fence fixture").unwrap();
let before = fs::read(&path).unwrap();
assert!(FsRuntimeStore::open_or_create(root, "runtime-test").is_err());
assert_eq!(fs::read(&path).unwrap(), before);
}
}
#[test] #[test]
fn schema_v10_startup_migrates_stopped_sessions_atomically() { fn schema_v10_startup_migrates_stopped_sessions_atomically() {
for corrupt in [false, true] { for corrupt in [false, true] {
@@ -2761,7 +3010,7 @@ mod tests {
assert_eq!(record["schema_version"], 10); assert_eq!(record["schema_version"], 10);
} else { } else {
drop(result.unwrap()); drop(result.unwrap());
assert_eq!(stored_schema_version(&root), 11); assert_eq!(stored_schema_version(&root), u64::from(SCHEMA_VERSION));
let manifest: serde_json::Value = let manifest: serde_json::Value =
read_json(&session.join("session.json"), "test").unwrap(); read_json(&session.join("session.json"), "test").unwrap();
assert_eq!(manifest["schema_version"], 4); assert_eq!(manifest["schema_version"], 4);
@@ -3151,7 +3400,7 @@ mod tests {
let temp = tempfile::tempdir().unwrap(); let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store"); let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root); write_empty_schema_v8_store(&root);
let staging = migration_sibling(&root, "schema-v11-staging").unwrap(); let staging = migration_sibling(&root, "schema-v12-staging").unwrap();
fs::create_dir_all(&staging).unwrap(); fs::create_dir_all(&staging).unwrap();
fs::write(staging.join("partial"), b"partial").unwrap(); fs::write(staging.join("partial"), b"partial").unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -3166,7 +3415,7 @@ mod tests {
let temp = tempfile::tempdir().unwrap(); let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store"); let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root); write_empty_schema_v8_store(&root);
let backup = migration_sibling(&root, "pre-schema-v11-backup").unwrap(); let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap();
fs::rename(&root, &backup).unwrap(); fs::rename(&root, &backup).unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -3180,10 +3429,10 @@ mod tests {
let temp = tempfile::tempdir().unwrap(); let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store"); let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root); write_empty_schema_v8_store(&root);
let staging = migration_sibling(&root, "schema-v11-staging").unwrap(); let staging = migration_sibling(&root, "schema-v12-staging").unwrap();
copy_runtime_tree(&root, &root, &staging).unwrap(); copy_runtime_tree(&root, &root, &staging).unwrap();
migrate_runtime_store_in_place(&staging, "test-runtime").unwrap(); migrate_runtime_store_in_place(&staging, "test-runtime").unwrap();
let backup = migration_sibling(&root, "pre-schema-v11-backup").unwrap(); let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap();
fs::rename(&root, &backup).unwrap(); fs::rename(&root, &backup).unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -3198,10 +3447,10 @@ mod tests {
let temp = tempfile::tempdir().unwrap(); let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store"); let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root); write_empty_schema_v8_store(&root);
let staging = migration_sibling(&root, "schema-v11-staging").unwrap(); let staging = migration_sibling(&root, "schema-v12-staging").unwrap();
fs::create_dir_all(&staging).unwrap(); fs::create_dir_all(&staging).unwrap();
fs::write(staging.join(RUNTIME_FILE), b"{\"schema_version\":8}").unwrap(); fs::write(staging.join(RUNTIME_FILE), b"{\"schema_version\":8}").unwrap();
let backup = migration_sibling(&root, "pre-schema-v11-backup").unwrap(); let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap();
fs::rename(&root, &backup).unwrap(); fs::rename(&root, &backup).unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -3218,7 +3467,7 @@ mod tests {
let root = temp.path().join("runtime-store"); let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root); write_empty_schema_v8_store(&root);
migrate_runtime_store_in_place(&root, "test-runtime").unwrap(); migrate_runtime_store_in_place(&root, "test-runtime").unwrap();
let backup = migration_sibling(&root, "pre-schema-v11-backup").unwrap(); let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap();
write_empty_schema_v8_store(&backup); write_empty_schema_v8_store(&backup);
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -3228,14 +3477,21 @@ mod tests {
#[test] #[test]
fn earlier_migration_backup_cannot_be_mistaken_for_a_fresh_runtime() { fn earlier_migration_backup_cannot_be_mistaken_for_a_fresh_runtime() {
let temp = tempfile::tempdir().unwrap(); for suffix in [
let root = temp.path().join("runtime-store"); "pre-schema-v9-backup",
let backup = migration_sibling(&root, "pre-schema-v9-backup").unwrap(); "pre-schema-v10-backup",
write_empty_schema_v8_store(&backup); "schema-v11-staging",
let error = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap_err(); "pre-schema-v11-backup",
assert!(error.to_string().contains("explicit recovery")); ] {
assert!(!root.exists()); let temp = tempfile::tempdir().unwrap();
assert!(backup.join(RUNTIME_FILE).exists()); let root = temp.path().join("runtime-store");
let backup = migration_sibling(&root, suffix).unwrap();
write_empty_schema_v8_store(&backup);
let error = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap_err();
assert!(error.to_string().contains("explicit recovery"));
assert!(!root.exists());
assert!(backup.join(RUNTIME_FILE).exists());
}
} }
#[test] #[test]
+24
View File
@@ -2102,6 +2102,9 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
{ {
return Some("workers:input"); return Some("workers:input");
} }
if path == "/v1/subject-create-recovery" && *method == Method::POST {
return Some("workers:delete");
}
if path.ends_with("/stop") if path.ends_with("/stop")
|| path.ends_with("/cancel") || path.ends_with("/cancel")
|| (path == "/v1/subject-release" && *method == Method::POST) || (path == "/v1/subject-release" && *method == Method::POST)
@@ -2973,6 +2976,19 @@ mod tests {
} }
} }
#[test]
fn subject_create_recovery_requires_worker_delete_permission() {
assert_eq!(
required_runtime_permission(&Method::POST, "/v1/subject-create-recovery"),
Some("workers:delete")
);
// Existing ordinary release authorization is unchanged.
assert_eq!(
required_runtime_permission(&Method::POST, "/v1/subject-release"),
Some("workers:stop")
);
}
#[test] #[test]
fn retention_routes_require_worker_delete_permission() { fn retention_routes_require_worker_delete_permission() {
assert_eq!( assert_eq!(
@@ -3197,6 +3213,14 @@ mod tests {
struct AcceptingBackend; struct AcceptingBackend;
impl WorkerExecutionBackend for AcceptingBackend { impl WorkerExecutionBackend for AcceptingBackend {
fn recover_subject_create(
&self,
_request: &runtime_api::SubjectCreateRecoveryRequest,
) -> Result<(), String> {
// This test backend owns no real executions.
Ok(())
}
fn backend_id(&self) -> &str { fn backend_id(&self) -> &str {
"http-test" "http-test"
} }
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
} }
} }
async fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, runtime_api::RuntimeApiError> {
let scope = material_scope(&self.state, &request.workspace_id)?;
self.state
.runtime
.recover_subject_create_scoped(&scope, request)
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)
}
async fn subject_release( async fn subject_release(
&self, &self,
request: runtime_api::SubjectReleaseRequest, request: runtime_api::SubjectReleaseRequest,
@@ -1086,6 +1086,54 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
} }
#[tokio::test]
async fn subject_create_recovery_signed_http_requires_delete_and_matching_workspace() {
let f = Fixture::new();
let identity = RuntimeIdentityMaterial::generate("recovery-key").unwrap();
let trust = workspace_trust(&identity);
let app = runtime_http_router_with_workspace_auth(
f.runtime.clone(),
None,
workspace_test_auth(vec![trust.clone()]),
);
let path = "/v1/subject-create-recovery";
let mut request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: f.scope.workspace_id.clone(),
worker_id: WorkerId::now_v7(),
create_fingerprint: "unpublished-create".into(),
operation_id: "recovery-1".into(),
};
for operation in ["workers:read", "workers:stop", "workers:create"] {
let body = serde_json::to_vec(&request).unwrap();
let mut claims = signed_claims(&trust, "POST", path, &body);
claims.operation = operation.into();
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
request.workspace_id = "other-workspace".into();
let body = serde_json::to_vec(&request).unwrap();
let claims = signed_claims(&trust, "POST", path, &body);
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
request.workspace_id = f.scope.workspace_id.clone();
for _ in 0..2 {
let body = serde_json::to_vec(&request).unwrap();
let claims = signed_claims(&trust, "POST", path, &body);
assert_eq!(claims.operation, "workers:delete");
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::OK);
let bytes = axum::body::to_bytes(response.into_body(), 16384)
.await
.unwrap();
let receipt: runtime_api::SubjectCreateRecoveryResponse =
serde_json::from_slice(&bytes).unwrap();
assert_eq!(receipt.worker_id, request.worker_id);
assert_eq!(receipt.workspace_id, request.workspace_id);
assert_eq!(receipt.create_fingerprint, request.create_fingerprint);
assert_eq!(receipt.operation_id, request.operation_id);
}
}
#[tokio::test] #[tokio::test]
async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() { async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() {
let f = Fixture::new(); let f = Fixture::new();
+333 -3
View File
@@ -355,7 +355,92 @@ impl Drop for WorkerOperationLease {
} }
} }
/// Irreversible admission exclusion, independent of the removable Worker aggregate.
/// A saved response is historical: retries still require a fresh backend absence check.
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct SubjectCreateRecoveryFence {
pub(crate) request: runtime_api::SubjectCreateRecoveryRequest,
pub(crate) response: Option<runtime_api::SubjectCreateRecoveryResponse>,
}
impl Runtime { impl Runtime {
pub fn recover_subject_create_scoped(
&self,
scope: &RuntimeWorkspaceScope,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, RuntimeError> {
if request.workspace_id != scope.workspace_id
|| request.create_fingerprint.trim().is_empty()
|| request.operation_id.trim().is_empty()
|| request.create_fingerprint.len() > 512
|| request.operation_id.len() > 512
{
return Err(RuntimeError::InvalidRequest(
"invalid Subject create recovery identity".into(),
));
}
let lease = self.worker_operation_lock(request.worker_id)?;
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
let backend = {
let mut state = self.lock()?;
state.ensure_running()?;
// A memory-only Runtime cannot issue durable proof.
if state.backend == RuntimeBackendKind::Memory {
return Err(RuntimeError::InvalidRequest(
"Subject create recovery requires durable storage".into(),
));
}
state.ensure_workspace_owner(scope, false)?;
if state.workers.contains_key(&request.worker_id) {
return Err(RuntimeError::InvalidRequest(
"Subject create catalog record is still present".into(),
));
}
if let Some(saved) = state.subject_create_recovery_fences.get(&request.worker_id) {
if saved.request != request {
return Err(RuntimeError::InvalidRequest(
"different Subject create recovery is retained".into(),
));
}
} else {
state.ensure_workspace_owner(scope, true)?;
state.subject_create_recovery_fences.insert(
request.worker_id,
SubjectCreateRecoveryFence {
request: request.clone(),
response: None,
},
);
}
// Never roll back admission exclusion, including uncertain persistence failures.
// Persist it before even checking backend availability.
state.persist_runtime_snapshot()?;
state.execution_backend.clone().ok_or_else(|| {
RuntimeError::InvalidRequest(
"Subject create execution absence cannot be confirmed".into(),
)
})?
};
backend
.recover_subject_create(&request)
.map_err(RuntimeError::InvalidRequest)?;
let response = runtime_api::SubjectCreateRecoveryResponse {
workspace_id: request.workspace_id.clone(),
worker_id: request.worker_id,
create_fingerprint: request.create_fingerprint.clone(),
operation_id: request.operation_id.clone(),
};
let mut state = self.lock()?;
state
.subject_create_recovery_fences
.get_mut(&request.worker_id)
.expect("recovery fence retained under Worker operation lock")
.response = Some(response.clone());
state.persist_runtime_snapshot()?;
Ok(response)
}
/// Reserves the actual Controller while the Host reconciles final saved Journal material. /// Reserves the actual Controller while the Host reconciles final saved Journal material.
/// A Runtime restart loses proof of a reservation, never reconstructs it from Worker status. /// A Runtime restart loses proof of a reservation, never reconstructs it from Worker status.
pub fn subject_release_scoped( pub fn subject_release_scoped(
@@ -1262,6 +1347,8 @@ impl Runtime {
let _operation_guard = operation_lock let _operation_guard = operation_lock
.lock() .lock()
.map_err(|_| RuntimeError::StatePoisoned)?; .map_err(|_| RuntimeError::StatePoisoned)?;
self.lock()?
.ensure_subject_create_not_fenced(request.worker_id)?;
if let Some(existing) = self.existing_worker_for_create(&request, scope)? { if let Some(existing) = self.existing_worker_for_create(&request, scope)? {
return Ok(existing); return Ok(existing);
} }
@@ -5482,6 +5569,7 @@ struct RuntimeState {
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
next_diagnostic_id: u64, next_diagnostic_id: u64,
workers: BTreeMap<WorkerId, WorkerRecord>, workers: BTreeMap<WorkerId, WorkerRecord>,
subject_create_recovery_fences: BTreeMap<WorkerId, SubjectCreateRecoveryFence>,
subject_release_operations: BTreeMap< subject_release_operations: BTreeMap<
(WorkerId, String), (WorkerId, String),
( (
@@ -5519,6 +5607,7 @@ impl RuntimeState {
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
next_diagnostic_id: 1, next_diagnostic_id: 1,
workers: BTreeMap::new(), workers: BTreeMap::new(),
subject_create_recovery_fences: BTreeMap::new(),
subject_release_operations: BTreeMap::new(), subject_release_operations: BTreeMap::new(),
workdir_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(),
workspace_owners: BTreeMap::new(), workspace_owners: BTreeMap::new(),
@@ -5551,6 +5640,7 @@ impl RuntimeState {
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
next_diagnostic_id: 1, next_diagnostic_id: 1,
workers: BTreeMap::new(), workers: BTreeMap::new(),
subject_create_recovery_fences: BTreeMap::new(),
subject_release_operations: BTreeMap::new(), subject_release_operations: BTreeMap::new(),
workdir_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(),
workspace_owners: BTreeMap::new(), workspace_owners: BTreeMap::new(),
@@ -5697,6 +5787,7 @@ impl RuntimeState {
workspace_backend_resource_clients: BTreeMap::new(), workspace_backend_resource_clients: BTreeMap::new(),
next_diagnostic_id, next_diagnostic_id,
workers, workers,
subject_create_recovery_fences: persisted.subject_create_recovery_fences,
subject_release_operations: persisted subject_release_operations: persisted
.subject_releases .subject_releases
.into_iter() .into_iter()
@@ -5727,6 +5818,7 @@ impl RuntimeState {
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
fn persisted_state(&self) -> PersistedRuntimeState { fn persisted_state(&self) -> PersistedRuntimeState {
PersistedRuntimeState { PersistedRuntimeState {
subject_create_recovery_fences: self.subject_create_recovery_fences.clone(),
subject_releases: self.subject_release_operations.values().cloned().collect(), subject_releases: self.subject_release_operations.values().cloned().collect(),
display_name: self.display_name.clone(), display_name: self.display_name.clone(),
status: self.status, status: self.status,
@@ -5871,11 +5963,21 @@ impl RuntimeState {
Ok(()) Ok(())
} }
fn ensure_subject_create_not_fenced(&self, worker_id: WorkerId) -> Result<(), RuntimeError> {
if self.subject_create_recovery_fences.contains_key(&worker_id) {
return Err(RuntimeError::InvalidRequest(
"Subject create identity is irreversibly fenced".into(),
));
}
Ok(())
}
fn ensure_subject_not_reserved( fn ensure_subject_not_reserved(
&self, &self,
worker: &WorkerRef, worker: &WorkerRef,
allow_disconnected: bool, allow_disconnected: bool,
) -> Result<(), RuntimeError> { ) -> Result<(), RuntimeError> {
self.ensure_subject_create_not_fenced(worker.worker_id)?;
use runtime_api::SubjectReleaseState as State; use runtime_api::SubjectReleaseState as State;
if self if self
.subject_release_operations .subject_release_operations
@@ -5945,6 +6047,13 @@ impl RuntimeState {
} }
fn forget_workspace_owner_if_unused(&mut self, workspace_id: &str) -> bool { fn forget_workspace_owner_if_unused(&mut self, workspace_id: &str) -> bool {
if self
.subject_create_recovery_fences
.values()
.any(|fence| fence.request.workspace_id == workspace_id)
{
return false;
}
if self if self
.workers .workers
.values() .values()
@@ -8293,6 +8402,8 @@ mod tests {
#[derive(Default)] #[derive(Default)]
struct TestExecutionBackend { struct TestExecutionBackend {
recovery_absent: AtomicBool,
recovery_checks: std::sync::atomic::AtomicUsize,
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>, spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>, job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>,
dispatch_result: Mutex<Option<WorkerExecutionResult>>, dispatch_result: Mutex<Option<WorkerExecutionResult>>,
@@ -8367,6 +8478,18 @@ mod tests {
} }
impl WorkerExecutionBackend for TestExecutionBackend { impl WorkerExecutionBackend for TestExecutionBackend {
fn recover_subject_create(
&self,
_request: &runtime_api::SubjectCreateRecoveryRequest,
) -> Result<(), String> {
self.recovery_checks.fetch_add(1, Ordering::SeqCst);
if self.recovery_absent.load(Ordering::SeqCst) {
Ok(())
} else {
Err("execution absence is unconfirmed".into())
}
}
fn subject_release( fn subject_release(
&self, &self,
request: &runtime_api::SubjectReleaseRequest, request: &runtime_api::SubjectReleaseRequest,
@@ -13098,7 +13221,7 @@ mod tests {
}) })
.unwrap_err(); .unwrap_err();
assert!(error.to_string().contains( assert!(error.to_string().contains(
"unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, or 11" "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, or 12"
)); ));
let _ = std::fs::remove_dir_all(root); let _ = std::fs::remove_dir_all(root);
@@ -13140,7 +13263,7 @@ mod tests {
let worker_aggregate: serde_json::Value = let worker_aggregate: serde_json::Value =
serde_json::from_slice(&std::fs::read(worker_store_dir.join("worker.json")).unwrap()) serde_json::from_slice(&std::fs::read(worker_store_dir.join("worker.json")).unwrap())
.unwrap(); .unwrap();
assert_eq!(worker_aggregate["schema_version"], serde_json::json!(11)); assert_eq!(worker_aggregate["schema_version"], serde_json::json!(12));
assert_eq!(worker_aggregate["status"], serde_json::json!("stopped")); assert_eq!(worker_aggregate["status"], serde_json::json!("stopped"));
assert_eq!( assert_eq!(
worker_aggregate["execution_state"]["state"], worker_aggregate["execution_state"]["state"],
@@ -13280,6 +13403,213 @@ mod tests {
let _ = std::fs::remove_dir_all(root); let _ = std::fs::remove_dir_all(root);
} }
#[cfg(feature = "fs-store")]
#[test]
fn subject_create_recovery_is_durable_exact_and_rechecks_absence() {
let root = tempfile::tempdir().unwrap();
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
.with_runtime_id("test-runtime");
let scoped = scope("workspace-a", "server-a");
let backend = Arc::new(TestExecutionBackend::default());
backend.recovery_absent.store(true, Ordering::SeqCst);
let runtime =
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
let create = scoped_task_request("unpublished create", "workspace-a");
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: scoped.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "recovery-1".into(),
};
runtime.store_config_bundle(test_bundle()).unwrap();
*backend.spawn_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::RolledBack(
WorkerExecutionResult::rejected(
WorkerExecutionOperation::Spawn,
"create failed before publication",
),
));
assert!(
runtime
.create_worker_scoped(&scoped, create.clone())
.is_err()
);
assert!(
!runtime
.lock()
.unwrap()
.workers
.contains_key(&create.worker_id)
);
let response = runtime
.recover_subject_create_scoped(&scoped, request.clone())
.unwrap();
assert_eq!(response.worker_id, request.worker_id);
assert!(
runtime
.create_worker_scoped(&scoped, create.clone())
.is_err()
);
assert!(
!runtime
.lock()
.unwrap()
.forget_workspace_owner_if_unused("workspace-a")
);
assert_eq!(
runtime
.lock()
.unwrap()
.workspace_owners
.get("workspace-a")
.map(String::as_str),
Some("server-a")
);
assert_eq!(
runtime
.recover_subject_create_scoped(&scoped, request.clone())
.unwrap(),
response
);
assert_eq!(backend.recovery_checks.load(Ordering::SeqCst), 2);
for mut mismatch in [request.clone(), request.clone(), request.clone()]
.into_iter()
.enumerate()
{
match mismatch.0 {
0 => mismatch.1.create_fingerprint.push_str("-wrong"),
1 => mismatch.1.operation_id.push_str("-wrong"),
_ => mismatch.1.workspace_id.push_str("-wrong"),
}
assert!(
runtime
.recover_subject_create_scoped(&scoped, mismatch.1)
.is_err()
);
}
assert!(
runtime
.recover_subject_create_scoped(
&scope("workspace-a", "other-server"),
request.clone()
)
.is_err()
);
drop(runtime);
let backend = Arc::new(TestExecutionBackend::default());
let reopened =
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
assert!(reopened.create_worker_scoped(&scoped, create).is_err());
assert!(
reopened
.recover_subject_create_scoped(&scoped, request.clone())
.is_err(),
"saved success cannot replace fresh absence proof"
);
backend.recovery_absent.store(true, Ordering::SeqCst);
assert_eq!(
reopened
.recover_subject_create_scoped(&scoped, request)
.unwrap(),
response
);
assert_eq!(backend.recovery_checks.load(Ordering::SeqCst), 2);
}
#[cfg(feature = "fs-store")]
#[test]
fn subject_create_recovery_unknown_backend_retains_fence_and_owner() {
let root = tempfile::tempdir().unwrap();
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
.with_runtime_id("test-runtime");
let scoped = scope("workspace-a", "server-a");
let runtime = Runtime::with_fs_store(options.clone()).unwrap();
let create = scoped_task_request("unpublished create", "workspace-a");
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: scoped.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "recovery-unknown".into(),
};
assert!(
runtime
.recover_subject_create_scoped(&scoped, request.clone())
.is_err()
);
assert!(
runtime
.lock()
.unwrap()
.subject_create_recovery_fences
.get(&request.worker_id)
.unwrap()
.response
.is_none()
);
drop(runtime);
let backend = Arc::new(TestExecutionBackend::default());
let reopened =
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
assert!(reopened.create_worker_scoped(&scoped, create).is_err());
assert!(
reopened
.recover_subject_create_scoped(
&scope("workspace-a", "other-server"),
request.clone()
)
.is_err()
);
assert!(
reopened
.recover_subject_create_scoped(&scoped, request.clone())
.is_err()
);
backend.recovery_absent.store(true, Ordering::SeqCst);
reopened
.recover_subject_create_scoped(&scoped, request)
.unwrap();
}
#[cfg(feature = "fs-store")]
#[test]
fn subject_create_recovery_rejects_existing_catalog_even_when_stopped() {
let root = tempfile::tempdir().unwrap();
let backend = Arc::new(TestExecutionBackend::default());
backend.recovery_absent.store(true, Ordering::SeqCst);
let runtime = Runtime::with_fs_store_and_execution_backend(
crate::fs_store::FsRuntimeStoreOptions::new(root.path())
.with_runtime_id("test-runtime"),
backend.clone(),
)
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let scoped = scope("workspace-a", "server-a");
let create = scoped_task_request("existing subject", "workspace-a");
let worker = runtime
.create_worker_scoped(&scoped, create.clone())
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: scoped.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "not-unpublished".into(),
};
assert!(
runtime
.recover_subject_create_scoped(&scoped, request)
.is_err()
);
assert_eq!(backend.recovery_checks.load(Ordering::SeqCst), 0);
assert!(
runtime
.lock()
.unwrap()
.subject_create_recovery_fences
.is_empty()
);
}
// Persistence component test; real Controller reservation has separate coverage. // Persistence component test; real Controller reservation has separate coverage.
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
#[test] #[test]
@@ -14164,7 +14494,7 @@ mod tests {
); );
let migrated_aggregate: serde_json::Value = let migrated_aggregate: serde_json::Value =
serde_json::from_slice(&std::fs::read(&worker_path).unwrap()).unwrap(); serde_json::from_slice(&std::fs::read(&worker_path).unwrap()).unwrap();
assert_eq!(migrated_aggregate["schema_version"], serde_json::json!(11)); assert_eq!(migrated_aggregate["schema_version"], serde_json::json!(12));
assert!(migrated_aggregate.get("request").is_none()); assert!(migrated_aggregate.get("request").is_none());
assert!(migrated_aggregate.get("execution").is_none()); assert!(migrated_aggregate.get("execution").is_none());
assert_eq!( assert_eq!(
@@ -3369,6 +3369,42 @@ where
&self.backend_id &self.backend_id
} }
fn recover_subject_create(
&self,
request: &runtime_api::SubjectCreateRecoveryRequest,
) -> Result<(), String> {
let worker_ref = WorkerRef::new(request.worker_id);
let lock = self.worker_lock(&worker_ref)?;
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
if self
.runtime
.lock()
.map_err(|_| "adapter runtime poisoned")?
.is_none()
{
return Err("execution backend is unavailable".into());
}
if self
.pending_restores
.lock()
.map_err(|_| "restore registry poisoned")?
.contains_key(&worker_ref)
{
return Err("Subject create has a pending execution candidate".into());
}
if self
.workers
.lock()
.map_err(|_| "worker registry poisoned")?
.contains_key(&worker_ref)
{
// Even a closed protocol or stopped snapshot is residual execution,
// not absence. Recovery does not stop or remove anything.
return Err("Subject create execution is still present".into());
}
Ok(())
}
fn subject_release( fn subject_release(
&self, &self,
request: &runtime_api::SubjectReleaseRequest, request: &runtime_api::SubjectReleaseRequest,
@@ -6069,6 +6105,48 @@ mod tests {
assert!(backend.workers.lock().unwrap().is_empty()); assert!(backend.workers.lock().unwrap().is_empty());
} }
#[test]
fn subject_create_recovery_checks_actual_absence_and_pending_candidates() {
let root = tempfile::tempdir().unwrap();
let backend = WorkerRuntimeExecutionBackend::new(
ProfileRuntimeWorkerFactory::new(root.path())
.with_runtime_store_dir(root.path().join("runtime")),
)
.unwrap();
let worker_ref = WorkerRef::new(WorkerId::now_v7());
let recovery = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: "ws-test".into(),
worker_id: worker_ref.worker_id,
create_fingerprint: "failed-create".into(),
operation_id: "recovery-1".into(),
};
backend.recover_subject_create(&recovery).unwrap();
// No Controller exists, but unresolved task ownership is not absence.
backend.pending_restores.lock().unwrap().insert(
worker_ref.clone(),
Arc::new(PendingRuntimeRestore {
request: direct_restore_request(worker_ref.clone()),
task: tokio::sync::Mutex::new(None),
result: Mutex::new(None),
failure: Mutex::new(Some("task completion unknown".into())),
}),
);
assert!(
backend
.recover_subject_create(&recovery)
.unwrap_err()
.contains("pending")
);
assert!(
backend
.pending_restores
.lock()
.unwrap()
.contains_key(&worker_ref)
);
assert!(backend.workers.lock().unwrap().is_empty());
}
#[test] #[test]
fn pending_restore_join_failure_never_becomes_absence_on_stop_retry() { fn pending_restore_join_failure_never_becomes_absence_on_stop_retry() {
let root = tempfile::tempdir().unwrap(); let root = tempfile::tempdir().unwrap();
@@ -6127,6 +6205,18 @@ mod tests {
fn stop_joins_healthy_self_terminated_controller_with_closed_endpoint() { fn stop_joins_healthy_self_terminated_controller_with_closed_endpoint() {
let root = tempfile::tempdir().unwrap(); let root = tempfile::tempdir().unwrap();
let (backend, request) = profile_stop_fixture(root.path()); let (backend, request) = profile_stop_fixture(root.path());
let recovery = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: "ws-test".into(),
worker_id: request.worker_ref.worker_id,
create_fingerprint: "failed-create".into(),
operation_id: "recovery-1".into(),
};
assert!(
backend
.recover_subject_create(&recovery)
.unwrap_err()
.contains("still present")
);
let execution = backend let execution = backend
.workers .workers
.lock() .lock()
@@ -6167,8 +6257,16 @@ mod tests {
}) })
.unwrap(); .unwrap();
assert!(!execution.shutdown_requested.load(Ordering::Acquire)); assert!(!execution.shutdown_requested.load(Ordering::Acquire));
assert!(
backend
.recover_subject_create(&recovery)
.unwrap_err()
.contains("still present"),
"closed Controller is still residual execution"
);
assert!(backend.stop_worker(&request.worker_ref).is_accepted()); assert!(backend.stop_worker(&request.worker_ref).is_accepted());
assert!(backend.workers.lock().unwrap().is_empty()); assert!(backend.workers.lock().unwrap().is_empty());
backend.recover_subject_create(&recovery).unwrap();
assert!(execution.tasks.tasks.lock().unwrap().is_empty()); assert!(execution.tasks.tasks.lock().unwrap().is_empty());
assert!(backend.stop_worker(&request.worker_ref).is_accepted()); assert!(backend.stop_worker(&request.worker_ref).is_accepted());
} }
+21
View File
@@ -11298,6 +11298,7 @@ mod build_summary_prompt_tests {
enum AttributionOutcome { enum AttributionOutcome {
Success, Success,
Rejected, Rejected,
Unauthorized,
TransportUnknown, TransportUnknown,
} }
@@ -11344,6 +11345,16 @@ mod build_summary_prompt_tests {
AttributionOutcome::TransportUnknown => Err(WorkspaceClientError::Request( AttributionOutcome::TransportUnknown => Err(WorkspaceClientError::Request(
"connection closed after request send".to_string(), "connection closed after request send".to_string(),
)), )),
AttributionOutcome::Unauthorized => Ok(WorkspaceResponse {
status: 401,
body: serde_json::to_string(&server_api::RepositoryApiError::new(
401,
"Unauthorized",
"invalid runtime request proof",
Vec::new(),
))
.unwrap(),
}),
AttributionOutcome::Rejected => Ok(WorkspaceResponse { AttributionOutcome::Rejected => Ok(WorkspaceResponse {
status: 503, status: 503,
body: "temporarily unavailable".to_string(), body: "temporarily unavailable".to_string(),
@@ -11576,6 +11587,7 @@ mod build_summary_prompt_tests {
for (first, expected_state) in [ for (first, expected_state) in [
(AttributionOutcome::TransportUnknown, "outcome_unknown"), (AttributionOutcome::TransportUnknown, "outcome_unknown"),
(AttributionOutcome::Rejected, "failed"), (AttributionOutcome::Rejected, "failed"),
(AttributionOutcome::Unauthorized, "failed"),
] { ] {
let client = Arc::new(AttributionLifecycleClient::new([ let client = Arc::new(AttributionLifecycleClient::new([
first, first,
@@ -11589,6 +11601,15 @@ mod build_summary_prompt_tests {
) )
.unwrap_err(); .unwrap_err();
assert!(error.to_string().contains(expected_state)); assert!(error.to_string().contains(expected_state));
if matches!(first, AttributionOutcome::Unauthorized) {
assert!(error.to_string().contains("rejected with HTTP 401"));
assert!(!error.to_string().contains("outcome_unknown"));
assert_eq!(
client.requests.lock().unwrap().len(),
1,
"no automatic retry after auth rejection"
);
}
let incomplete = store let incomplete = store
.read_by_name(&worker.manifest().worker.name) .read_by_name(&worker.manifest().worker.name)
.unwrap() .unwrap()
@@ -475,7 +475,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
(98, "Assignment-independent implementation reports".into()), (98, "Assignment-independent implementation reports".into()),
(99, "Explicit hosting administrators and self-registration policy".into()), (99, "Explicit hosting administrators and self-registration policy".into()),
(100, "Provisional users and scoped passkey activation".into()), (100, "Provisional users and scoped passkey activation".into()),
(101, "Durable explicit Subject body release".into()) (101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into())
] ]
); );
for (table, rows) in retained.iter().zip(original.iter()) { for (table, rows) in retained.iter().zip(original.iter()) {
+96 -1
View File
@@ -1288,6 +1288,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
}) })
} }
fn recover_subject_create(
&self,
_request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, String> {
Err("Subject create recovery fencing is unavailable on this Runtime".into())
}
fn subject_release( fn subject_release(
&self, &self,
_request: runtime_api::SubjectReleaseRequest, _request: runtime_api::SubjectReleaseRequest,
@@ -2251,6 +2258,21 @@ impl RuntimeRegistry {
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id) runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
} }
pub fn recover_subject_create(
&self,
runtime_id: &str,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.recover_subject_create(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "recover_subject_create_unproven".into(),
message,
})
}
pub fn subject_release( pub fn subject_release(
&self, &self,
runtime_id: &str, runtime_id: &str,
@@ -3319,6 +3341,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
}) })
} }
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, String> {
if request.workspace_id != self.workspace_id {
return Err("Subject create recovery Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
self.runtime
.recover_subject_create_scoped(&scope, request)
.map_err(|e| e.to_string())
}
fn subject_release( fn subject_release(
&self, &self,
request: runtime_api::SubjectReleaseRequest, request: runtime_api::SubjectReleaseRequest,
@@ -4164,7 +4199,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
if path.ends_with("/completions") { if path.ends_with("/completions") {
return "workers:read"; return "workers:read";
} }
if path.contains("/retention/") || (path.starts_with("/v1/workers/") && method == "DELETE") { if path.contains("/retention/")
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|| (path == "/v1/subject-create-recovery" && method == "POST")
{
return "workers:delete"; return "workers:delete";
} }
if path.starts_with("/v1/session-public/") && method == "POST" { if path.starts_with("/v1/session-public/") && method == "POST" {
@@ -5561,6 +5599,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}) })
} }
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.recover_subject_create(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn subject_release( fn subject_release(
&self, &self,
request: runtime_api::SubjectReleaseRequest, request: runtime_api::SubjectReleaseRequest,
@@ -8377,6 +8427,51 @@ mod tests {
); );
} }
#[test]
fn remote_subject_create_recovery_uses_generated_management_transport() {
assert_eq!(
workspace_runtime_operation("POST", "/v1/subject-create-recovery"),
"workers:delete"
);
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: "workspace-test".into(),
worker_id: EmbeddedWorkerId::now_v7(),
create_fingerprint: "sha256:failed-create".into(),
operation_id: "recover-1".into(),
};
// Remote adapter transport only; signed Runtime authorization is tested
// in worker-runtime's actual HTTP router suite.
let (base_url, server) = serve_mock_http(vec![mock_response(
"POST",
"/v1/subject-create-recovery",
true,
200,
serde_json::to_string(&request).unwrap(),
)]);
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(
RemoteRuntimeConfig::new(
"remote:recover",
"Recovery transport fixture",
base_url,
Some("secret-token".into()),
),
"workspace-test".into(),
"http://127.0.0.1:8787".into(),
)
.unwrap(),
);
let response = registry
.recover_subject_create("remote:recover", request.clone())
.unwrap();
assert_eq!(response.workspace_id, request.workspace_id);
assert_eq!(response.worker_id, request.worker_id);
assert_eq!(response.create_fingerprint, request.create_fingerprint);
assert_eq!(response.operation_id, request.operation_id);
server.join().unwrap();
}
#[test] #[test]
fn remote_session_erasure_uses_the_generated_management_transport() { fn remote_session_erasure_uses_the_generated_management_transport() {
// The loopback fixture substitutes only the remote Runtime. This covers // The loopback fixture substitutes only the remote Runtime. This covers
@@ -381,6 +381,8 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_results;
DROP TABLE ticket_implementation_delegations; DROP TABLE ticket_implementation_delegations;
DROP TABLE merge_request_source_observations; DROP TABLE merge_request_source_observations;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases; DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101; UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations; DROP TABLE user_activations;
@@ -515,6 +517,8 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
// Reconstruct the prior composition before its new business capture existed. // Reconstruct the prior composition before its new business capture existed.
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases; DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101; UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations; DROP TABLE user_activations;
@@ -675,7 +679,9 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
if historical { if historical {
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TABLE subject_worker_releases; DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101; UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations; DROP TABLE user_activations;
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100; UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
@@ -699,6 +699,19 @@ CREATE TABLE worker_create_reservations (
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
CHECK ((singleton_key IS NULL) = (singleton_generation IS NULL)) CHECK ((singleton_key IS NULL) = (singleton_generation IS NULL))
); );
CREATE TABLE subject_create_recoveries (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
singleton_key TEXT NOT NULL,
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
operation_id TEXT NOT NULL CHECK (length(operation_id) > 0),
create_fingerprint TEXT NOT NULL,
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
PRIMARY KEY (workspace_id, operation_id),
UNIQUE (workspace_id, runtime_id, worker_id, singleton_key, singleton_generation),
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
);
CREATE TABLE subject_worker_releases ( CREATE TABLE subject_worker_releases (
workspace_id TEXT NOT NULL, workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL, runtime_id TEXT NOT NULL,
+41 -9
View File
@@ -2934,10 +2934,10 @@ fn repository_api_rejection(path: &str, status: StatusCode, message: &str) -> Re
if path.starts_with("/api/w/") && path.contains("/drive/") { if path.starts_with("/api/w/") && path.contains("/drive/") {
return (status, Json(drive::failure(status.as_u16()))).into_response(); return (status, Json(drive::failure(status.as_u16()))).into_response();
} }
if path == "/api/repositories" // Generated Workspace API clients share this error envelope, not only
|| path.starts_with("/api/repositories/") // repository endpoints. A bare 401 here becomes a response-decode failure
|| (path.starts_with("/api/w/") && path.contains("/repositories")) // and incorrectly turns a known auth rejection into an unknown outcome.
{ if path.starts_with("/api/") {
return ( return (
status, status,
Json(server_api::RepositoryApiError::new( Json(server_api::RepositoryApiError::new(
@@ -5484,6 +5484,12 @@ impl WorkspaceApi {
let subject = singleton_key let subject = singleton_key
.as_deref() .as_deref()
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX)); .and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
if let Some(subject) = subject
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
{
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
return Err(ApiError::with_diagnostics(error, diagnostics));
}
let subject_gate = subject let subject_gate = subject
.map(|s| subjektiv_journal::lifecycle_lock(self, s)) .map(|s| subjektiv_journal::lifecycle_lock(self, s))
.transpose()?; .transpose()?;
@@ -22372,15 +22378,12 @@ async fn scoped_record_subjektiv_session(
.run(move || { .run(move || {
validate_workspace_scope(&api, &path.workspace_id)?; validate_workspace_scope(&api, &path.workspace_id)?;
let attribution = if request.create_if_missing { let attribution = if request.create_if_missing {
let attribution = subjektiv_session_attribution( subjektiv_session_attribution(
&api, &api,
&path.workspace_id, &path.workspace_id,
&context, &context,
&request.session_id, &request.session_id,
)?; )?
open_subjektiv_store(&api)?
.record_session_attribution(attribution)
.map_err(|error| Error::InvalidInput(error.to_string()))?
} else { } else {
require_existing_subjektiv_session_attribution( require_existing_subjektiv_session_attribution(
&api, &api,
@@ -47939,6 +47942,9 @@ mod tests {
>, >,
materializer: worker_runtime::working_directory::RuntimeGitMaterializer, materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
spawn_failure: std::sync::Mutex<Option<String>>, spawn_failure: std::sync::Mutex<Option<String>>,
recovery_failure: std::sync::Mutex<Option<String>>,
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
input_failure: std::sync::Mutex<Option<String>>, input_failure: std::sync::Mutex<Option<String>>,
cancel_failure: std::sync::Mutex<Option<String>>, cancel_failure: std::sync::Mutex<Option<String>>,
input_hook: std::sync::Mutex< input_hook: std::sync::Mutex<
@@ -47974,6 +47980,9 @@ mod tests {
std::env::temp_dir().join(unique), std::env::temp_dir().join(unique),
), ),
spawn_failure: std::sync::Mutex::new(None), spawn_failure: std::sync::Mutex::new(None),
recovery_failure: std::sync::Mutex::new(None),
recovery_requests: std::sync::Mutex::new(Vec::new()),
recovery_hook: std::sync::Mutex::new(None),
input_failure: std::sync::Mutex::new(None), input_failure: std::sync::Mutex::new(None),
cancel_failure: std::sync::Mutex::new(None), cancel_failure: std::sync::Mutex::new(None),
input_hook: std::sync::Mutex::new(None), input_hook: std::sync::Mutex::new(None),
@@ -48068,6 +48077,29 @@ mod tests {
"deterministic-workspace-server-test" "deterministic-workspace-server-test"
} }
fn recover_subject_create(
&self,
request: &runtime_api::SubjectCreateRecoveryRequest,
) -> std::result::Result<(), String> {
self.recovery_requests.lock().unwrap().push(request.clone());
if let Some(hook) = self.recovery_hook.lock().unwrap().take() {
hook();
}
if let Some(error) = self.recovery_failure.lock().unwrap().take() {
return Err(error);
}
if self
.contexts
.lock()
.unwrap()
.keys()
.any(|worker| worker.worker_id == request.worker_id)
{
return Err("actual execution remains".into());
}
Ok(())
}
fn create_working_directory( fn create_working_directory(
&self, &self,
request: &worker_runtime::catalog::WorkingDirectoryRequest, request: &worker_runtime::catalog::WorkingDirectoryRequest,
@@ -346,6 +346,11 @@ pub(super) fn bind_host_session(
} }
} }
} }
// Publication (including attribution) shares the recovery gate. A callback
// authenticated before a recovery intent must not leave a late attribution.
let attr = store
.record_session_attribution(attr)
.map_err(|error| Error::InvalidInput(error.to_string()))?;
let writer = store let writer = store
.claim_journal_writer(&attr.subject_id, &attr.runtime_id, &attr.worker_id) .claim_journal_writer(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?; .map_err(domain)?;
@@ -386,6 +391,93 @@ pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<(
Ok(()) Ok(())
} }
/// Explicit creation can reconcile an earlier failed, never-published body.
/// No missing catalog/status observation is treated as a Runtime fencing receipt.
pub(super) fn recover_failed_subject_create(api: &WorkspaceApi, subject: &str) -> Result<()> {
let continuation = lifecycle_lock(api, &format!("release-continuation:{subject}"))?;
let _continuation = continuation
.lock()
.map_err(|_| Error::Store("release continuation poisoned".into()))?;
let gate = lifecycle_lock(api, subject)?;
let subject_key = crate::subjektiv::subject_worker_singleton_key(subject)
.map_err(|e| Error::InvalidInput(e.to_string()))?;
let store = store(api)?;
let (lease, request) = {
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
let Some(lease) = api
.config_store
.current_worker_singleton_owner(api.workspace_id(), &subject_key)?
else {
return Ok(());
};
if api
.config_store
.subject_create_recovery_candidate(api.workspace_id(), &lease)?
.is_none()
{
return Ok(());
}
if !store
.journal_body_is_unpublished(subject, &lease.worker.runtime_id, &lease.worker.worker_id)
.map_err(domain)?
{
// Published history must go through the ordinary continuity release.
return Ok(());
}
let operation_id = format!(
"subject-create-recovery:{}:{}",
lease.worker.worker_id, lease.generation
);
let fingerprint = api.config_store.prepare_subject_create_recovery(
api.workspace_id(),
&lease,
&operation_id,
)?;
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: api.workspace_id().into(),
worker_id: worker_runtime::identity::WorkerId::parse(&lease.worker.worker_id)
.ok_or_else(|| Error::Store("invalid retained Worker ID".into()))?,
operation_id,
create_fingerprint: fingerprint,
};
(lease, request)
};
// Never hold the callback lane/gate across Runtime I/O. Unknown outcome
// leaves the durable intent in place and reuses the exact identity next time.
let response = api
.runtime
.recover_subject_create(&lease.worker.runtime_id, request.clone())
.map_err(RuntimeRegistryError::into_error)?;
if response.workspace_id != request.workspace_id
|| response.worker_id != request.worker_id
|| response.operation_id != request.operation_id
|| response.create_fingerprint != request.create_fingerprint
{
return Err(Error::RepositoryConflict(
"Subject create recovery acknowledgment mismatch; retain occupancy".into(),
));
}
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
if !store
.journal_body_is_unpublished(subject, &lease.worker.runtime_id, &lease.worker.worker_id)
.map_err(domain)?
{
return Err(Error::RepositoryConflict(
"Subject publication appeared during recovery; retain occupancy".into(),
));
}
api.config_store.complete_subject_create_recovery(
api.workspace_id(),
&lease,
&request.operation_id,
&request.create_fingerprint,
)
}
pub(super) fn lifecycle_lock(api: &WorkspaceApi, subject: &str) -> Result<Arc<Mutex<()>>> { pub(super) fn lifecycle_lock(api: &WorkspaceApi, subject: &str) -> Result<Arc<Mutex<()>>> {
let mut locks = api let mut locks = api
.journal_lifecycle .journal_lifecycle
@@ -197,7 +197,20 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
.unwrap(); .unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{reason}"); assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{reason}");
let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert!(response_body.is_empty(), "{response_body:?}"); let error: server_api::RepositoryApiError =
serde_json::from_slice(&response_body).unwrap();
assert_eq!(error.error, "Unauthorized");
assert_eq!(error.message, "invalid runtime request proof");
assert_eq!(
serde_json::from_slice::<Value>(&response_body).unwrap(),
serde_json::to_value(server_api::RepositoryApiError::new(
401,
"Unauthorized",
"invalid runtime request proof",
Vec::new()
))
.unwrap()
);
let (event, text) = logs.event(); let (event, text) = logs.event();
assert_eq!(event["reason"], reason); assert_eq!(event["reason"], reason);
assert_eq!(event["proof_verification"], verification); assert_eq!(event["proof_verification"], verification);
@@ -338,6 +351,124 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m
} }
} }
#[tokio::test]
async fn common_api_rejections_use_json_without_changing_drive_or_non_api_contracts() {
for status in [
StatusCode::BAD_REQUEST,
StatusCode::UNAUTHORIZED,
StatusCode::FORBIDDEN,
StatusCode::NOT_FOUND,
StatusCode::CONFLICT,
StatusCode::SERVICE_UNAVAILABLE,
] {
for path in [
"/api/w/ws/subjektiv/sessions",
"/api/w/ws/workers/self/workdir-session/operations",
"/api/runtime/v1/workspaces/ws/resources/fetch",
"/api/repositories",
] {
let response = repository_api_rejection(path, status, "request rejected");
assert_eq!(response.status(), status);
assert_eq!(response.headers()[CONTENT_TYPE], "application/json");
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(
serde_json::from_slice::<Value>(&bytes).unwrap(),
serde_json::to_value(server_api::RepositoryApiError::new(
status.as_u16(),
status.canonical_reason().unwrap(),
"request rejected",
Vec::new()
))
.unwrap()
);
}
let drive = repository_api_rejection("/api/w/ws/drive/mutate", status, "request rejected");
assert_eq!(drive.status(), status);
assert_eq!(
serde_json::from_slice::<Value>(
&to_bytes(drive.into_body(), usize::MAX).await.unwrap()
)
.unwrap(),
serde_json::to_value(drive::failure(status.as_u16())).unwrap()
);
let shell = repository_api_rejection("/w/ws", status, "request rejected");
assert_eq!(shell.status(), status);
assert!(
to_bytes(shell.into_body(), usize::MAX)
.await
.unwrap()
.is_empty()
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn subjektiv_auth_rejection_survives_runtime_generated_client_on_both_routers() {
use worker::WorkspaceClient;
struct AbortServer(tokio::task::JoinHandle<()>);
impl Drop for AbortServer {
fn drop(&mut self) {
self.0.abort();
}
}
let workspace = tempfile::tempdir().unwrap();
let mut api = test_api(workspace.path()).await;
let trusted = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
configure_runtime_request_auth(&mut api, &trusted, "runtime-test");
let untrusted = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
for app in [
build_router(api.clone()),
workspace_server_router(WorkspaceServerApi::new_for_test(
api.config.clone(),
api.store.clone(),
)),
] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let _server = AbortServer(tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
}));
// Exercise missing authentication, signature rejection and a valid proof
// whose Worker has no catalog membership/reservation.
for identity in [None, Some(&untrusted), Some(&trusted)] {
let client = worker_runtime::worker_source::RuntimeOwnedWorkspaceClient::new(
TEST_WORKSPACE_ID,
&url,
"runtime-test",
"unregistered-worker",
);
let client = match identity {
Some(identity) => client.with_runtime_request_source(identity, "server-test"),
None => client,
};
let response = tokio::task::spawn_blocking(move || {
client.execute_server_operation(worker::WorkspaceServerOperation::SubjektivRecordSession(
server_api::SubjektivRecordSessionRequest {
session_id: session_store::new_session_id().to_string(), create_if_missing: true,
},
))
}).await.unwrap().expect("an explicit HTTP rejection must remain a WorkspaceResponse, not an unknown transport outcome");
assert_eq!(response.status, 401);
let error: server_api::RepositoryApiError =
serde_json::from_str(&response.body).unwrap();
assert_eq!(error.error, "Unauthorized");
assert_eq!(
error.message,
if identity.is_some() {
"invalid runtime request proof"
} else {
"authentication required"
}
);
assert!(!response.body.contains("unregistered-worker"));
if let Some(identity) = identity {
assert!(!response.body.contains(&identity.private_key));
}
}
}
}
#[tokio::test] #[tokio::test]
async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() { async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() {
let workspace = tempfile::tempdir().unwrap(); let workspace = tempfile::tempdir().unwrap();
@@ -330,7 +330,7 @@ async fn rejected_create_releases_key_but_retains_subject_occupancy_without_rele
.reserve_worker_create( .reserve_worker_create(
TEST_WORKSPACE_ID, TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID, EMBEDDED_RUNTIME_ID,
"rejected-create", "manual:rejected-create",
&"f".repeat(64), &"f".repeat(64),
Some("subjektiv:rejected-create"), Some("subjektiv:rejected-create"),
&memory, &memory,
@@ -378,5 +378,274 @@ async fn rejected_create_releases_key_but_retains_subject_occupancy_without_rele
.worker, .worker,
worker, worker,
); );
// Retained Subject occupancy is not a live create reservation. A new
// manual request must not send the terminal Worker's identity to Runtime.
let retry = api.config_store.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID,
"manual:retry-rejected-create",
&"f".repeat(64),
Some("subjektiv:rejected-create"),
&memory,
);
assert!(
matches!(retry, Err(Error::RepositoryConflict(ref message))
if message.contains("worker_singleton_release_required")),
"terminal reservation must fail before Runtime dispatch: {retry:?}"
);
} }
} }
async fn unpublished_recovery_fixture(
root: &std::path::Path,
) -> (
WorkspaceApi,
Arc<DeterministicExecutionBackend>,
String,
crate::store::WorkerCreateReservation,
) {
let (api, backend) = test_api_with_recording_backend(root).await;
let subject = open_subjektiv_store(&api)
.unwrap()
.create_subject(crate::subjektiv::SubjectRole::new("recovery-test").unwrap())
.unwrap();
let key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
let memory = api
.config_store
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
.unwrap();
let old = api
.config_store
.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID,
"manual:failed-subject",
&"f".repeat(64),
Some(&key),
&memory,
)
.unwrap();
api.config_store
.fail_worker_create_reservation(
TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID,
old.worker_id,
&old.create_fingerprint,
)
.unwrap();
(api, backend, subject.id, old)
}
fn recovery_spawn_request(subject: &str) -> WorkerSpawnRequest {
serde_json::from_value(json!({
"intent": WorkerSpawnIntent::WorkspaceCompanion,
"acceptance": WorkerSpawnAcceptanceRequirement::RunAccepted { expected_segments: 0 },
"profile": worker_runtime::catalog::ProfileSelector::Builtin("builtin:companion".into()),
"singleton_key": crate::subjektiv::subject_worker_singleton_key(subject).unwrap(),
}))
.unwrap()
}
#[tokio::test]
async fn unpublished_create_recovery_retries_same_fence_then_creates_new_worker() {
let root = tempfile::tempdir().unwrap();
let (api, backend, subject, old) = unpublished_recovery_fixture(root.path()).await;
let old_worker = RuntimeWorkerRef::new(EMBEDDED_RUNTIME_ID, old.worker_id.to_string());
let lease = old.singleton.as_ref().unwrap();
let late = crate::subjektiv::SubjectSessionAttribution::new(
&subject,
EMBEDDED_RUNTIME_ID,
old.worker_id.to_string(),
session_store::new_session_id().to_string(),
)
.unwrap();
let late_session = late.session_id.clone();
// Models an already-authorized callback arriving while Runtime fencing is
// in flight. The Host intent must reject it BEFORE attribution insertion.
let callback_api = api.clone();
*backend.recovery_hook.lock().unwrap() = Some(Box::new(move || {
assert!(subjektiv_journal::bind_host_session(&callback_api, late).is_err());
}));
*backend.recovery_failure.lock().unwrap() = Some("unconfirmed execution absence".into());
assert!(
api.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject))
.is_err()
);
assert!(
api.config_store
.subject_release_pending(TEST_WORKSPACE_ID, &old_worker)
.unwrap()
);
assert_eq!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap(),
Some(lease.clone())
);
let store = open_subjektiv_store(&api).unwrap();
assert!(store.session_attribution(&late_session).unwrap().is_none());
assert!(
store
.journal_body_is_unpublished(&subject, EMBEDDED_RUNTIME_ID, &old_worker.worker_id)
.unwrap()
);
let replacement = api
.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject))
.unwrap();
assert_eq!(replacement.state, InternalWorkerOperationState::Accepted);
let replacement = replacement.worker.unwrap().worker;
assert_ne!(replacement, old_worker);
let current = api
.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap()
.unwrap();
assert_eq!(current.worker, replacement);
assert_eq!(current.generation, lease.generation + 1);
let requests = backend.recovery_requests.lock().unwrap();
assert_eq!(requests.len(), 2);
assert_eq!(
requests[0], requests[1],
"unknown result must reuse exact recovery identity"
);
// A stale completion cannot release the replacement generation.
api.config_store
.complete_subject_create_recovery(
TEST_WORKSPACE_ID,
lease,
&requests[0].operation_id,
&old.create_fingerprint,
)
.unwrap();
assert_eq!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap(),
Some(current)
);
}
#[tokio::test]
async fn unpublished_create_recovery_refuses_any_attribution_or_orphan_writer() {
for orphan_writer in [false, true] {
let root = tempfile::tempdir().unwrap();
let (api, backend, subject, old) = unpublished_recovery_fixture(root.path()).await;
let store = open_subjektiv_store(&api).unwrap();
if orphan_writer {
store
.claim_journal_writer(&subject, EMBEDDED_RUNTIME_ID, &old.worker_id.to_string())
.unwrap();
} else {
store
.record_session_attribution(
crate::subjektiv::SubjectSessionAttribution::new(
&subject,
EMBEDDED_RUNTIME_ID,
old.worker_id.to_string(),
session_store::new_session_id().to_string(),
)
.unwrap(),
)
.unwrap();
}
assert!(
api.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject))
.is_err()
);
assert!(backend.recovery_requests.lock().unwrap().is_empty());
assert_eq!(
api.config_store
.current_worker_singleton_owner(
TEST_WORKSPACE_ID,
&old.singleton.as_ref().unwrap().key
)
.unwrap(),
old.singleton
);
}
}
#[tokio::test]
async fn unpublished_create_recovery_resumes_after_runtime_proof_before_host_commit() {
let root = tempfile::tempdir().unwrap();
let (api, backend, subject, old) = unpublished_recovery_fixture(root.path()).await;
let lease = old.singleton.as_ref().unwrap();
let op = format!(
"subject-create-recovery:{}:{}",
old.worker_id, lease.generation
);
let fingerprint = api
.config_store
.prepare_subject_create_recovery(TEST_WORKSPACE_ID, lease, &op)
.unwrap();
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: TEST_WORKSPACE_ID.into(),
worker_id: old.worker_id,
create_fingerprint: fingerprint,
operation_id: op,
};
api.runtime
.recover_subject_create(EMBEDDED_RUNTIME_ID, request)
.unwrap();
// Simulate loss of the receipt before Host completion. The Host must not
// infer completion from missing catalog state, and can retry the exact op.
assert_eq!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap(),
old.singleton
);
subjektiv_journal::recover_failed_subject_create(&api, &subject).unwrap();
assert!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap()
.is_none()
);
assert_eq!(backend.recovery_requests.lock().unwrap().len(), 2);
}
#[tokio::test]
async fn failed_subject_spawn_can_recover_on_next_explicit_create() {
let root = tempfile::tempdir().unwrap();
let (api, backend) = test_api_with_recording_backend(root.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("failed-spawn").unwrap())
.unwrap();
let key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
*backend.spawn_failure.lock().unwrap() =
Some("Session registration failed before publication".into());
let rejected = api
.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject.id))
.unwrap();
assert_eq!(rejected.state, InternalWorkerOperationState::Rejected);
let old = api
.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
.unwrap()
.unwrap();
assert!(
api.config_store
.subject_create_recovery_candidate(TEST_WORKSPACE_ID, &old)
.unwrap()
.is_some()
);
assert!(
backend.recovery_requests.lock().unwrap().is_empty(),
"failure alone must not release occupancy"
);
let result = api
.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject.id))
.unwrap();
assert_eq!(result.state, InternalWorkerOperationState::Accepted);
let current = api
.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
.unwrap()
.unwrap();
assert_ne!(current.worker, old.worker);
assert_eq!(current.generation, old.generation + 1);
assert_eq!(backend.recovery_requests.lock().unwrap().len(), 1);
}
+591 -6
View File
@@ -119,7 +119,7 @@ pub use ticket::{
}; };
const OLDEST_SCHEMA_VERSION: i64 = 50; const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 101; const LATEST_SCHEMA_VERSION: i64 = 102;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -432,6 +432,11 @@ const MIGRATIONS: &[Migration] = &[
name: "Durable explicit Subject body release", name: "Durable explicit Subject body release",
apply: migrate_subject_release_v100_to_v101, apply: migrate_subject_release_v100_to_v101,
}, },
Migration {
version: 102,
name: "Durable failed Subject creation recovery",
apply: migrate_subject_create_recovery_v101_to_v102,
},
]; ];
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
@@ -2997,7 +3002,7 @@ impl SqliteWorkspaceStore {
.query_row( .query_row(
"SELECT allocation_key, request_fingerprint, create_fingerprint, \ "SELECT allocation_key, request_fingerprint, create_fingerprint, \
memory_language, singleton_key, \ memory_language, singleton_key, \
singleton_generation \ singleton_generation, state \
FROM worker_create_reservations \ FROM worker_create_reservations \
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3",
params![ params![
@@ -3013,11 +3018,21 @@ impl SqliteWorkspaceStore {
row.get::<_, Option<String>>(3)?, row.get::<_, Option<String>>(3)?,
row.get::<_, Option<String>>(4)?, row.get::<_, Option<String>>(4)?,
row.get::<_, Option<i64>>(5)?, row.get::<_, Option<i64>>(5)?,
row.get::<_, String>(6)?,
)) ))
}, },
) )
.optional()?; .optional()?;
if let Some((stored_allocation_key, stored_request_fingerprint, create_fingerprint, language, stored_singleton_key, stored_generation)) = stored // Subject occupancy survives removal until an explicit release
// proof. It must not resurrect the removed creation reservation:
// Runtime callbacks require a live reservation or catalog member.
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
return Err(Error::RepositoryConflict(format!(
"worker_singleton_release_required: Worker {}:{} has a terminal create reservation but still owns the Subject; complete Subject release before creating a replacement",
owner.worker.runtime_id, owner.worker.worker_id
)));
}
if let Some((stored_allocation_key, stored_request_fingerprint, create_fingerprint, language, stored_singleton_key, stored_generation, _)) = stored
&& allocation_key.starts_with("manual:") && allocation_key.starts_with("manual:")
&& stored_allocation_key.starts_with("manual:") && stored_allocation_key.starts_with("manual:")
&& owner.worker.runtime_id == runtime_id && owner.worker.runtime_id == runtime_id
@@ -3199,6 +3214,99 @@ impl SqliteWorkspaceStore {
}) })
} }
/// Eligibility is Backend evidence only. The Host must separately prove that Subjektiv
/// never published this body and obtain the Runtime's exact create-fence receipt.
pub(crate) fn subject_create_recovery_candidate(
&self,
workspace: &str,
lease: &WorkerSingletonLease,
) -> Result<Option<String>> {
self.with_conn(|conn| subject_create_recovery_candidate(conn, workspace, lease))
}
pub(crate) fn prepare_subject_create_recovery(
&self,
workspace: &str,
lease: &WorkerSingletonLease,
operation_id: &str,
) -> Result<String> {
if operation_id.trim().is_empty() || operation_id.len() > 512 {
return Err(Error::InvalidInput(
"invalid Subject create recovery operation".into(),
));
}
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let fingerprint = subject_create_recovery_candidate(&tx, workspace, lease)?
.ok_or_else(|| Error::RepositoryConflict("Subject create recovery requires the current removed reservation without a registry row".into()))?;
let saved: Option<(String, String, String, u64, String)> = tx.query_row(
"SELECT runtime_id,worker_id,singleton_key,singleton_generation,create_fingerprint FROM subject_create_recoveries WHERE workspace_id=?1 AND operation_id=?2",
params![workspace, operation_id],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)),
).optional()?;
if let Some(saved) = saved {
if saved != (lease.worker.runtime_id.clone(), lease.worker.worker_id.clone(), lease.key.clone(), lease.generation, fingerprint.clone()) {
return Err(Error::RepositoryConflict("Subject create recovery operation was used with different input".into()));
}
return Ok(fingerprint);
}
let busy: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_create_recoveries WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5)",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, lease.generation as i64],
|row| row.get(0),
)?;
if busy {
return Err(Error::RepositoryConflict("Subject create recovery already has a durable operation".into()));
}
tx.execute(
"INSERT INTO subject_create_recoveries(workspace_id,runtime_id,worker_id,singleton_key,singleton_generation,operation_id,create_fingerprint) VALUES(?1,?2,?3,?4,?5,?6,?7)",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, lease.generation as i64, operation_id, fingerprint],
)?;
tx.commit()?;
Ok(fingerprint)
})
}
/// Called only after the Host verifies the Runtime's typed receipt for this exact create.
/// The completed bit records that proof; it neither creates a writer binding nor rewrites
/// the reservation. An exact completed replay is a receipt, not authority over a successor.
pub(crate) fn complete_subject_create_recovery(
&self,
workspace: &str,
lease: &WorkerSingletonLease,
operation_id: &str,
create_fingerprint: &str,
) -> Result<()> {
let generation = i64::try_from(lease.generation)
.map_err(|_| Error::InvalidInput("invalid singleton generation".into()))?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let completed: Option<bool> = tx.query_row(
"SELECT completed FROM subject_create_recoveries WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND operation_id=?6 AND create_fingerprint=?7",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, generation, operation_id, create_fingerprint],
|row| row.get(0),
).optional()?;
match completed {
Some(true) => return Ok(()),
Some(false) => {},
None => return Err(Error::RepositoryConflict("Subject create recovery intent absent or mismatched".into())),
}
let current = subject_create_recovery_candidate(&tx, workspace, lease)?;
if current.as_deref() != Some(create_fingerprint) {
return Err(Error::RepositoryConflict("Subject create recovery reservation or singleton ownership changed".into()));
}
let changed = tx.execute(
"UPDATE subject_create_recoveries SET completed=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND operation_id=?6 AND create_fingerprint=?7 AND completed=0",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, generation, operation_id, create_fingerprint],
)?;
if changed != 1 {
return Err(Error::RepositoryConflict("Subject create recovery completion lost its exact intent".into()));
}
tx.commit()?;
Ok(())
})
}
pub(crate) fn retained_subject_release_operation( pub(crate) fn retained_subject_release_operation(
&self, &self,
workspace: &str, workspace: &str,
@@ -3213,7 +3321,7 @@ impl SqliteWorkspaceStore {
workspace: &str, workspace: &str,
worker: &RuntimeWorkerRef, worker: &RuntimeWorkerRef,
) -> Result<bool> { ) -> Result<bool> {
self.with_conn(|c| Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND cancelled=0 AND completed=0)",params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0))?)) self.with_conn(|c| Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND cancelled=0 AND completed=0) OR EXISTS(SELECT 1 FROM subject_create_recoveries WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND completed=0)",params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0))?))
} }
pub(crate) fn require_subject_release_intent( pub(crate) fn require_subject_release_intent(
@@ -15012,6 +15120,34 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
Ok(()) Ok(())
} }
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 101 {
return Err(Error::Store(
"Subject create recovery migration requires schema 101".into(),
));
}
let tx = conn.unchecked_transaction()?;
tx.execute_batch("CREATE TABLE subject_create_recoveries (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
singleton_key TEXT NOT NULL,
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
operation_id TEXT NOT NULL CHECK (length(operation_id) > 0),
create_fingerprint TEXT NOT NULL,
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
PRIMARY KEY (workspace_id, operation_id),
UNIQUE (workspace_id, runtime_id, worker_id, singleton_key, singleton_generation),
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
);")?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(102,?1)",
["Durable failed Subject creation recovery"],
)?;
tx.commit()?;
Ok(())
}
fn migrate_subject_release_v100_to_v101(conn: &Connection) -> Result<()> { fn migrate_subject_release_v100_to_v101(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 100 { if current_schema_version(conn)? != 100 {
return Err(Error::Store( return Err(Error::Store(
@@ -15702,6 +15838,33 @@ pub(crate) fn validate_worker_singleton_key(key: &str) -> Result<()> {
Ok(()) Ok(())
} }
fn subject_create_recovery_candidate(
conn: &Connection,
workspace: &str,
lease: &WorkerSingletonLease,
) -> Result<Option<String>> {
if !lease
.key
.starts_with(crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX)
{
return Err(Error::InvalidInput("not a Subject singleton".into()));
}
let generation = i64::try_from(lease.generation)
.map_err(|_| Error::InvalidInput("invalid singleton generation".into()))?;
Ok(conn.query_row(
"SELECT r.create_fingerprint FROM worker_create_reservations r
JOIN worker_singleton_owners o ON o.workspace_id=r.workspace_id
AND o.runtime_id=r.runtime_id AND o.worker_id=r.worker_id
AND o.singleton_key=r.singleton_key AND o.generation=r.singleton_generation
WHERE r.workspace_id=?1 AND r.runtime_id=?2 AND r.worker_id=?3
AND r.singleton_key=?4 AND r.singleton_generation=?5 AND r.state='removed'
AND NOT EXISTS(SELECT 1 FROM worker_registry w
WHERE w.workspace_id=r.workspace_id AND w.runtime_id=r.runtime_id AND w.worker_id=r.worker_id)",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, generation],
|row| row.get(0),
).optional()?)
}
fn current_worker_singleton_owner( fn current_worker_singleton_owner(
conn: &Connection, conn: &Connection,
workspace_id: &str, workspace_id: &str,
@@ -15712,6 +15875,7 @@ fn current_worker_singleton_owner(
"SELECT owner.runtime_id, owner.worker_id, owner.generation, \ "SELECT owner.runtime_id, owner.worker_id, owner.generation, \
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \ CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \ AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
THEN 'created' ELSE reservation.state END \ THEN 'created' ELSE reservation.state END \
FROM worker_singleton_owners owner \ FROM worker_singleton_owners owner \
LEFT JOIN worker_create_reservations reservation \ LEFT JOIN worker_create_reservations reservation \
@@ -18739,6 +18903,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
WorkspaceSchemaMigrationStep {version:99, name:"Explicit hosting administrators and self-registration policy".into()}, WorkspaceSchemaMigrationStep {version:99, name:"Explicit hosting administrators and self-registration policy".into()},
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()}, WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()}, WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
] ]
); );
@@ -18850,6 +19015,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(99, "Explicit hosting administrators and self-registration policy".to_string()), (99, "Explicit hosting administrators and self-registration policy".to_string()),
(100, "Provisional users and scoped passkey activation".to_string()), (100, "Provisional users and scoped passkey activation".to_string()),
(101, "Durable explicit Subject body release".to_string()), (101, "Durable explicit Subject body release".to_string()),
(102, "Durable failed Subject creation recovery".to_string()),
] ]
); );
assert!(!table_exists(conn, "trusted_runtime_records")?); assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -20907,6 +21073,47 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
conflict, conflict,
Error::RepositoryConflict(message) if message.contains("worker_singleton_owned") Error::RepositoryConflict(message) if message.contains("worker_singleton_owned")
)); ));
reopened
.fail_worker_create_reservation(
"workspace-a",
"runtime-a",
reserved.worker_id,
&reserved.create_fingerprint,
)
.unwrap();
drop(reopened);
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
let conflict = reopened
.reserve_worker_create(
"workspace-a",
"runtime-a",
"manual:retry-after-terminal-failure",
&stable_fingerprint,
Some("subjektiv:restart-retry"),
&memory,
)
.unwrap_err();
assert!(matches!(conflict, Error::RepositoryConflict(message)
if message.contains("worker_singleton_release_required")));
let worker = RuntimeWorkerRef::new("runtime-a", reserved.worker_id.to_string());
assert!(
!reopened
.has_active_worker_create_reservation("workspace-a", &worker)
.unwrap()
);
assert!(
reopened
.worker_resource_key("workspace-a", &worker)
.unwrap()
.is_none()
);
assert_eq!(
reopened
.current_worker_singleton_owner("workspace-a", "subjektiv:restart-retry")
.unwrap(),
reserved.singleton
);
} }
#[tokio::test] #[tokio::test]
@@ -21036,6 +21243,383 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
); );
} }
#[test]
fn subject_create_recovery_migration_is_atomic_and_matches_fresh_schema() {
let dir = tempfile::tempdir().unwrap();
let database = dir.path().join("server.db");
prepare_retained_schema(&database, 101);
let conn = Connection::open(&database).unwrap();
configure_sqlite(&conn).unwrap();
conn.execute_batch("CREATE TRIGGER interrupt_create_recovery BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=102 BEGIN SELECT RAISE(ABORT,'injected recovery migration interruption'); END;").unwrap();
assert!(migrate_subject_create_recovery_v101_to_v102(&conn).is_err());
assert_eq!(current_schema_version(&conn).unwrap(), 101);
assert!(!table_exists(&conn, "subject_create_recoveries").unwrap());
conn.execute_batch("DROP TRIGGER interrupt_create_recovery")
.unwrap();
migrate_subject_create_recovery_v101_to_v102(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 102);
assert!(migrate_subject_create_recovery_v101_to_v102(&conn).is_err());
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
let schema = |c: &Connection| -> Result<String> {
Ok(c.query_row(
"SELECT sql FROM sqlite_schema WHERE name='subject_create_recoveries'",
[],
|r| r.get::<_, String>(0),
)?
.split_whitespace()
.collect::<Vec<_>>()
.join(" "))
};
assert_eq!(schema(&conn).unwrap(), fresh.with_conn(schema).unwrap());
assert_eq!(
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r
.get::<_, i64>(
0
))
.unwrap(),
0
);
drop(conn);
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
reopened
.with_conn(|c| verify_schema_history(c, 102))
.unwrap();
}
#[tokio::test]
async fn subject_create_recovery_is_durable_exact_and_cannot_release_a_new_generation() {
for baseline in [None, Some(52), Some(101)] {
let dir = tempfile::tempdir().unwrap();
let database = dir.path().join("server.db");
if let Some(baseline) = baseline {
prepare_retained_schema(&database, baseline);
}
let store = SqliteWorkspaceStore::open_api_fixture(&database).unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace-a".into(),
owner_account_id: "owner-account".into(),
display_name: "Workspace A".into(),
state: "active".into(),
created_at: "2026-10-01T00:00:00Z".into(),
updated_at: "2026-10-01T00:00:00Z".into(),
})
.await
.unwrap();
let memory = store.get_workspace_memory_settings("workspace-a").unwrap();
let key = "subjektiv:failed-create";
let first = store
.reserve_worker_create(
"workspace-a",
"runtime-a",
"manual:first",
"sha256:first",
Some(key),
&memory,
)
.unwrap();
let lease = first.singleton.unwrap();
let fingerprint = first.create_fingerprint;
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &lease)
.unwrap(),
None
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.is_err()
);
store
.fail_worker_create_reservation(
"workspace-a",
"runtime-a",
first.worker_id,
&fingerprint,
)
.unwrap();
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &lease)
.unwrap(),
Some(fingerprint.clone())
);
assert!(
store
.prepare_subject_create_recovery("workspace-b", &lease, "recover-first")
.is_err()
);
for wrong in [
WorkerSingletonLease {
generation: lease.generation + 1,
..lease.clone()
},
WorkerSingletonLease {
key: "subjektiv:other".into(),
..lease.clone()
},
WorkerSingletonLease {
worker: RuntimeWorkerRef::new("runtime-b", &lease.worker.worker_id),
..lease.clone()
},
WorkerSingletonLease {
worker: RuntimeWorkerRef::new("runtime-a", "other-worker"),
..lease.clone()
},
] {
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &wrong)
.unwrap(),
None
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &wrong, "recover-first")
.is_err()
);
}
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
assert_eq!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.unwrap(),
fingerprint
);
assert!(
store
.subject_release_pending("workspace-a", &lease.worker)
.unwrap()
);
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
Some(lease.clone())
);
assert!(
store
.reserve_worker_create(
"workspace-a",
"runtime-a",
"manual:retry",
"sha256:first",
Some(key),
&memory
)
.is_err()
);
drop(store);
let store = SqliteWorkspaceStore::open_for_test(&database).unwrap();
assert_eq!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.unwrap(),
fingerprint
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "different-operation")
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
"different-fingerprint"
)
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"different-operation",
&fingerprint
)
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-b",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
// A registry row appearing after prepare must block both replay and completion.
store.with_conn(|c| {
c.execute("INSERT INTO worker_registry(workspace_id,runtime_id,worker_id,display_name,retention_state,created_at,updated_at) VALUES('workspace-a',?1,?2,'body','normal','now','now')", params![lease.worker.runtime_id, lease.worker.worker_id])?;
Ok(())
}).unwrap();
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &lease)
.unwrap(),
None
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
store.with_conn(|c| {
c.execute("DELETE FROM worker_registry WHERE workspace_id='workspace-a' AND runtime_id=?1 AND worker_id=?2", params![lease.worker.runtime_id, lease.worker.worker_id])?;
// Exercise a current-generation change while the operation is incomplete.
c.execute("UPDATE worker_singleton_owners SET generation=generation+1 WHERE workspace_id='workspace-a' AND singleton_key=?1", [key])?;
Ok(())
}).unwrap();
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
assert!(
store
.subject_release_pending("workspace-a", &lease.worker)
.unwrap()
);
store.with_conn(|c| {
c.execute("UPDATE worker_singleton_owners SET generation=?2 WHERE workspace_id='workspace-a' AND singleton_key=?1", params![key, lease.generation as i64])?;
Ok(())
}).unwrap();
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint,
)
.unwrap();
drop(store);
let store = SqliteWorkspaceStore::open_for_test(&database).unwrap();
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint,
)
.unwrap();
assert!(
!store
.subject_release_pending("workspace-a", &lease.worker)
.unwrap()
);
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
None
);
let next = store
.reserve_worker_create(
"workspace-a",
"runtime-b",
"manual:next",
"sha256:next",
Some(key),
&memory,
)
.unwrap();
let next_lease = next.singleton.unwrap();
assert_eq!(next_lease.generation, lease.generation + 1);
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint,
)
.unwrap();
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&next_lease,
"recover-first",
&fingerprint
)
.is_err()
);
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
Some(next_lease.clone())
);
// The old completed proof cannot release the next removed generation.
store
.fail_worker_create_reservation(
"workspace-a",
"runtime-b",
next.worker_id,
&next.create_fingerprint,
)
.unwrap();
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
Some(next_lease.clone())
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &next_lease, "recover-first")
.is_err()
);
assert_eq!(
store
.prepare_subject_create_recovery("workspace-a", &next_lease, "recover-next")
.unwrap(),
next.create_fingerprint
);
store.with_conn(|c| {
assert_eq!(c.query_row("SELECT count(*) FROM subject_worker_releases", [], |r| r.get::<_, i64>(0))?, 0);
assert_eq!(c.query_row("SELECT state FROM worker_create_reservations WHERE workspace_id='workspace-a' AND worker_id=?1", [&lease.worker.worker_id], |r| r.get::<_, String>(0))?, "removed");
assert!(c.execute("UPDATE subject_create_recoveries SET completed=2", []).is_err());
assert!(c.execute("UPDATE subject_create_recoveries SET singleton_generation=0", []).is_err());
assert_eq!(c.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r.get::<_, i64>(0))?, 0);
Ok(())
}).unwrap();
}
}
#[test] #[test]
fn subject_release_migration_rolls_back_schema_and_marker_then_reopens() { fn subject_release_migration_rolls_back_schema_and_marker_then_reopens() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
@@ -21056,7 +21640,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
drop(conn); drop(conn);
let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap(); let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap();
reopened reopened
.with_conn(|conn| verify_schema_history(conn, 101)) .with_conn(|conn| verify_schema_history(conn, LATEST_SCHEMA_VERSION))
.unwrap(); .unwrap();
} }
@@ -21636,7 +22220,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(98, "Assignment-independent implementation reports".into()), (98, "Assignment-independent implementation reports".into()),
(99, "Explicit hosting administrators and self-registration policy".into()), (99, "Explicit hosting administrators and self-registration policy".into()),
(100, "Provisional users and scoped passkey activation".into()), (100, "Provisional users and scoped passkey activation".into()),
(101, "Durable explicit Subject body release".into())]); (101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into())]);
for (table, rows) in unchanged.iter().zip(before.iter()) { for (table, rows) in unchanged.iter().zip(before.iter()) {
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}"); assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
} }
@@ -95,6 +95,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"workdir_removal_operations", "workdir_removal_operations",
"worker_control_grants", "worker_control_grants",
"subject_worker_releases", "subject_worker_releases",
"subject_create_recoveries",
"worker_create_reservations", "worker_create_reservations",
"worker_diagnostics_archives", "worker_diagnostics_archives",
"worker_mutation_source_proof_jtis", "worker_mutation_source_proof_jtis",