Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: recover unpublished Subject creation and preserve auth rejections

Browse Source
This commit is contained in:
Hare
2026-10-19 18:56:24 +09:00
parent c961f244ab
commit 8e40fae6f4
20 changed files with 2243 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/assignment_tracking_tests.rs
+2 -1
View File
@@ -475,7 +475,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
(98, "Assignment-independent implementation reports".into()),
(99, "Explicit hosting administrators and self-registration policy".into()),
(100, "Provisional users and scoped passkey activation".into()),
(101, "Durable explicit Subject body release".into())
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into())
]
);
for (table, rows) in retained.iter().zip(original.iter()) {
crates/workspace-server/src/hosts.rs
+96 -1
View File
@@ -1288,6 +1288,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
})
}
fn recover_subject_create(
&self,
_request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, String> {
Err("Subject create recovery fencing is unavailable on this Runtime".into())
}
fn subject_release(
&self,
_request: runtime_api::SubjectReleaseRequest,
@@ -2251,6 +2258,21 @@ impl RuntimeRegistry {
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
}
pub fn recover_subject_create(
&self,
runtime_id: &str,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.recover_subject_create(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "recover_subject_create_unproven".into(),
message,
})
}
pub fn subject_release(
&self,
runtime_id: &str,
@@ -3319,6 +3341,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
})
}
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, String> {
if request.workspace_id != self.workspace_id {
return Err("Subject create recovery Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
self.runtime
.recover_subject_create_scoped(&scope, request)
.map_err(|e| e.to_string())
}
fn subject_release(
&self,
request: runtime_api::SubjectReleaseRequest,
@@ -4164,7 +4199,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
if path.ends_with("/completions") {
return "workers:read";
}
if path.contains("/retention/") || (path.starts_with("/v1/workers/") && method == "DELETE") {
if path.contains("/retention/")
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|| (path == "/v1/subject-create-recovery" && method == "POST")
{
return "workers:delete";
}
if path.starts_with("/v1/session-public/") && method == "POST" {
@@ -5561,6 +5599,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
})
}
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
) -> Result<runtime_api::SubjectCreateRecoveryResponse, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.recover_subject_create(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn subject_release(
&self,
request: runtime_api::SubjectReleaseRequest,
@@ -8377,6 +8427,51 @@ mod tests {
);
}
#[test]
fn remote_subject_create_recovery_uses_generated_management_transport() {
assert_eq!(
workspace_runtime_operation("POST", "/v1/subject-create-recovery"),
"workers:delete"
);
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: "workspace-test".into(),
worker_id: EmbeddedWorkerId::now_v7(),
create_fingerprint: "sha256:failed-create".into(),
operation_id: "recover-1".into(),
};
// Remote adapter transport only; signed Runtime authorization is tested
// in worker-runtime's actual HTTP router suite.
let (base_url, server) = serve_mock_http(vec![mock_response(
"POST",
"/v1/subject-create-recovery",
true,
200,
serde_json::to_string(&request).unwrap(),
)]);
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(
RemoteRuntimeConfig::new(
"remote:recover",
"Recovery transport fixture",
base_url,
Some("secret-token".into()),
),
"workspace-test".into(),
"http://127.0.0.1:8787".into(),
)
.unwrap(),
);
let response = registry
.recover_subject_create("remote:recover", request.clone())
.unwrap();
assert_eq!(response.workspace_id, request.workspace_id);
assert_eq!(response.worker_id, request.worker_id);
assert_eq!(response.create_fingerprint, request.create_fingerprint);
assert_eq!(response.operation_id, request.operation_id);
server.join().unwrap();
}
#[test]
fn remote_session_erasure_uses_the_generated_management_transport() {
// The loopback fixture substitutes only the remote Runtime. This covers
crates/workspace-server/src/implementation_result_tests.rs
+7 -1
View File
@@ -381,6 +381,8 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
DROP TABLE ticket_implementation_results;
DROP TABLE ticket_implementation_delegations;
DROP TABLE merge_request_source_observations;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations;
@@ -515,6 +517,8 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
// Reconstruct the prior composition before its new business capture existed.
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations;
@@ -675,7 +679,9 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
if historical {
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TABLE subject_worker_releases;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations;
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
crates/workspace-server/src/latest_schema.sql
+13
View File
@@ -699,6 +699,19 @@ CREATE TABLE worker_create_reservations (
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
CHECK ((singleton_key IS NULL) = (singleton_generation IS NULL))
);
CREATE TABLE subject_create_recoveries (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
singleton_key TEXT NOT NULL,
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
operation_id TEXT NOT NULL CHECK (length(operation_id) > 0),
create_fingerprint TEXT NOT NULL,
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
PRIMARY KEY (workspace_id, operation_id),
UNIQUE (workspace_id, runtime_id, worker_id, singleton_key, singleton_generation),
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
);
CREATE TABLE subject_worker_releases (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
crates/workspace-server/src/server.rs
+41 -9
View File
@@ -2934,10 +2934,10 @@ fn repository_api_rejection(path: &str, status: StatusCode, message: &str) -> Re
if path.starts_with("/api/w/") && path.contains("/drive/") {
return (status, Json(drive::failure(status.as_u16()))).into_response();
}
if path == "/api/repositories"
|| path.starts_with("/api/repositories/")
|| (path.starts_with("/api/w/") && path.contains("/repositories"))
{
// Generated Workspace API clients share this error envelope, not only
// repository endpoints. A bare 401 here becomes a response-decode failure
// and incorrectly turns a known auth rejection into an unknown outcome.
if path.starts_with("/api/") {
return (
status,
Json(server_api::RepositoryApiError::new(
@@ -5484,6 +5484,12 @@ impl WorkspaceApi {
let subject = singleton_key
.as_deref()
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
if let Some(subject) = subject
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
{
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
return Err(ApiError::with_diagnostics(error, diagnostics));
}
let subject_gate = subject
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
.transpose()?;
@@ -22372,15 +22378,12 @@ async fn scoped_record_subjektiv_session(
.run(move || {
validate_workspace_scope(&api, &path.workspace_id)?;
let attribution = if request.create_if_missing {
let attribution = subjektiv_session_attribution(
subjektiv_session_attribution(
&api,
&path.workspace_id,
&context,
&request.session_id,
)?;
open_subjektiv_store(&api)?
.record_session_attribution(attribution)
.map_err(|error| Error::InvalidInput(error.to_string()))?
)?
} else {
require_existing_subjektiv_session_attribution(
&api,
@@ -47939,6 +47942,9 @@ mod tests {
>,
materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
spawn_failure: std::sync::Mutex<Option<String>>,
recovery_failure: std::sync::Mutex<Option<String>>,
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
input_failure: std::sync::Mutex<Option<String>>,
cancel_failure: std::sync::Mutex<Option<String>>,
input_hook: std::sync::Mutex<
@@ -47974,6 +47980,9 @@ mod tests {
std::env::temp_dir().join(unique),
),
spawn_failure: std::sync::Mutex::new(None),
recovery_failure: std::sync::Mutex::new(None),
recovery_requests: std::sync::Mutex::new(Vec::new()),
recovery_hook: std::sync::Mutex::new(None),
input_failure: std::sync::Mutex::new(None),
cancel_failure: std::sync::Mutex::new(None),
input_hook: std::sync::Mutex::new(None),
@@ -48068,6 +48077,29 @@ mod tests {
"deterministic-workspace-server-test"
}
fn recover_subject_create(
&self,
request: &runtime_api::SubjectCreateRecoveryRequest,
) -> std::result::Result<(), String> {
self.recovery_requests.lock().unwrap().push(request.clone());
if let Some(hook) = self.recovery_hook.lock().unwrap().take() {
hook();
}
if let Some(error) = self.recovery_failure.lock().unwrap().take() {
return Err(error);
}
if self
.contexts
.lock()
.unwrap()
.keys()
.any(|worker| worker.worker_id == request.worker_id)
{
return Err("actual execution remains".into());
}
Ok(())
}
fn create_working_directory(
&self,
request: &worker_runtime::catalog::WorkingDirectoryRequest,
crates/workspace-server/src/server/subjektiv_journal.rs
+92
View File
@@ -346,6 +346,11 @@ pub(super) fn bind_host_session(
}
}
}
// Publication (including attribution) shares the recovery gate. A callback
// authenticated before a recovery intent must not leave a late attribution.
let attr = store
.record_session_attribution(attr)
.map_err(|error| Error::InvalidInput(error.to_string()))?;
let writer = store
.claim_journal_writer(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?;
@@ -386,6 +391,93 @@ pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<(
Ok(())
}
/// Explicit creation can reconcile an earlier failed, never-published body.
/// No missing catalog/status observation is treated as a Runtime fencing receipt.
pub(super) fn recover_failed_subject_create(api: &WorkspaceApi, subject: &str) -> Result<()> {
let continuation = lifecycle_lock(api, &format!("release-continuation:{subject}"))?;
let _continuation = continuation
.lock()
.map_err(|_| Error::Store("release continuation poisoned".into()))?;
let gate = lifecycle_lock(api, subject)?;
let subject_key = crate::subjektiv::subject_worker_singleton_key(subject)
.map_err(|e| Error::InvalidInput(e.to_string()))?;
let store = store(api)?;
let (lease, request) = {
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
let Some(lease) = api
.config_store
.current_worker_singleton_owner(api.workspace_id(), &subject_key)?
else {
return Ok(());
};
if api
.config_store
.subject_create_recovery_candidate(api.workspace_id(), &lease)?
.is_none()
{
return Ok(());
}
if !store
.journal_body_is_unpublished(subject, &lease.worker.runtime_id, &lease.worker.worker_id)
.map_err(domain)?
{
// Published history must go through the ordinary continuity release.
return Ok(());
}
let operation_id = format!(
"subject-create-recovery:{}:{}",
lease.worker.worker_id, lease.generation
);
let fingerprint = api.config_store.prepare_subject_create_recovery(
api.workspace_id(),
&lease,
&operation_id,
)?;
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: api.workspace_id().into(),
worker_id: worker_runtime::identity::WorkerId::parse(&lease.worker.worker_id)
.ok_or_else(|| Error::Store("invalid retained Worker ID".into()))?,
operation_id,
create_fingerprint: fingerprint,
};
(lease, request)
};
// Never hold the callback lane/gate across Runtime I/O. Unknown outcome
// leaves the durable intent in place and reuses the exact identity next time.
let response = api
.runtime
.recover_subject_create(&lease.worker.runtime_id, request.clone())
.map_err(RuntimeRegistryError::into_error)?;
if response.workspace_id != request.workspace_id
|| response.worker_id != request.worker_id
|| response.operation_id != request.operation_id
|| response.create_fingerprint != request.create_fingerprint
{
return Err(Error::RepositoryConflict(
"Subject create recovery acknowledgment mismatch; retain occupancy".into(),
));
}
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
if !store
.journal_body_is_unpublished(subject, &lease.worker.runtime_id, &lease.worker.worker_id)
.map_err(domain)?
{
return Err(Error::RepositoryConflict(
"Subject publication appeared during recovery; retain occupancy".into(),
));
}
api.config_store.complete_subject_create_recovery(
api.workspace_id(),
&lease,
&request.operation_id,
&request.create_fingerprint,
)
}
pub(super) fn lifecycle_lock(api: &WorkspaceApi, subject: &str) -> Result<Arc<Mutex<()>>> {
let mut locks = api
.journal_lifecycle
crates/workspace-server/src/server/tests/auth_logging_tests.rs
+132 -1
View File
@@ -197,7 +197,20 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{reason}");
let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert!(response_body.is_empty(), "{response_body:?}");
let error: server_api::RepositoryApiError =
serde_json::from_slice(&response_body).unwrap();
assert_eq!(error.error, "Unauthorized");
assert_eq!(error.message, "invalid runtime request proof");
assert_eq!(
serde_json::from_slice::<Value>(&response_body).unwrap(),
serde_json::to_value(server_api::RepositoryApiError::new(
401,
"Unauthorized",
"invalid runtime request proof",
Vec::new()
))
.unwrap()
);
let (event, text) = logs.event();
assert_eq!(event["reason"], reason);
assert_eq!(event["proof_verification"], verification);
@@ -338,6 +351,124 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m
}
}
#[tokio::test]
async fn common_api_rejections_use_json_without_changing_drive_or_non_api_contracts() {
for status in [
StatusCode::BAD_REQUEST,
StatusCode::UNAUTHORIZED,
StatusCode::FORBIDDEN,
StatusCode::NOT_FOUND,
StatusCode::CONFLICT,
StatusCode::SERVICE_UNAVAILABLE,
] {
for path in [
"/api/w/ws/subjektiv/sessions",
"/api/w/ws/workers/self/workdir-session/operations",
"/api/runtime/v1/workspaces/ws/resources/fetch",
"/api/repositories",
] {
let response = repository_api_rejection(path, status, "request rejected");
assert_eq!(response.status(), status);
assert_eq!(response.headers()[CONTENT_TYPE], "application/json");
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(
serde_json::from_slice::<Value>(&bytes).unwrap(),
serde_json::to_value(server_api::RepositoryApiError::new(
status.as_u16(),
status.canonical_reason().unwrap(),
"request rejected",
Vec::new()
))
.unwrap()
);
}
let drive = repository_api_rejection("/api/w/ws/drive/mutate", status, "request rejected");
assert_eq!(drive.status(), status);
assert_eq!(
serde_json::from_slice::<Value>(
&to_bytes(drive.into_body(), usize::MAX).await.unwrap()
)
.unwrap(),
serde_json::to_value(drive::failure(status.as_u16())).unwrap()
);
let shell = repository_api_rejection("/w/ws", status, "request rejected");
assert_eq!(shell.status(), status);
assert!(
to_bytes(shell.into_body(), usize::MAX)
.await
.unwrap()
.is_empty()
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn subjektiv_auth_rejection_survives_runtime_generated_client_on_both_routers() {
use worker::WorkspaceClient;
struct AbortServer(tokio::task::JoinHandle<()>);
impl Drop for AbortServer {
fn drop(&mut self) {
self.0.abort();
}
}
let workspace = tempfile::tempdir().unwrap();
let mut api = test_api(workspace.path()).await;
let trusted = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
configure_runtime_request_auth(&mut api, &trusted, "runtime-test");
let untrusted = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
for app in [
build_router(api.clone()),
workspace_server_router(WorkspaceServerApi::new_for_test(
api.config.clone(),
api.store.clone(),
)),
] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let _server = AbortServer(tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
}));
// Exercise missing authentication, signature rejection and a valid proof
// whose Worker has no catalog membership/reservation.
for identity in [None, Some(&untrusted), Some(&trusted)] {
let client = worker_runtime::worker_source::RuntimeOwnedWorkspaceClient::new(
TEST_WORKSPACE_ID,
&url,
"runtime-test",
"unregistered-worker",
);
let client = match identity {
Some(identity) => client.with_runtime_request_source(identity, "server-test"),
None => client,
};
let response = tokio::task::spawn_blocking(move || {
client.execute_server_operation(worker::WorkspaceServerOperation::SubjektivRecordSession(
server_api::SubjektivRecordSessionRequest {
session_id: session_store::new_session_id().to_string(), create_if_missing: true,
},
))
}).await.unwrap().expect("an explicit HTTP rejection must remain a WorkspaceResponse, not an unknown transport outcome");
assert_eq!(response.status, 401);
let error: server_api::RepositoryApiError =
serde_json::from_str(&response.body).unwrap();
assert_eq!(error.error, "Unauthorized");
assert_eq!(
error.message,
if identity.is_some() {
"invalid runtime request proof"
} else {
"authentication required"
}
);
assert!(!response.body.contains("unregistered-worker"));
if let Some(identity) = identity {
assert!(!response.body.contains(&identity.private_key));
}
}
}
}
#[tokio::test]
async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() {
let workspace = tempfile::tempdir().unwrap();
crates/workspace-server/src/server/tests/subject_spawn_tests.rs
+270 -1
View File
@@ -330,7 +330,7 @@ async fn rejected_create_releases_key_but_retains_subject_occupancy_without_rele
.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID,
"rejected-create",
"manual:rejected-create",
&"f".repeat(64),
Some("subjektiv:rejected-create"),
&memory,
@@ -378,5 +378,274 @@ async fn rejected_create_releases_key_but_retains_subject_occupancy_without_rele
.worker,
worker,
);
// Retained Subject occupancy is not a live create reservation. A new
// manual request must not send the terminal Worker's identity to Runtime.
let retry = api.config_store.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID,
"manual:retry-rejected-create",
&"f".repeat(64),
Some("subjektiv:rejected-create"),
&memory,
);
assert!(
matches!(retry, Err(Error::RepositoryConflict(ref message))
if message.contains("worker_singleton_release_required")),
"terminal reservation must fail before Runtime dispatch: {retry:?}"
);
}
}
async fn unpublished_recovery_fixture(
root: &std::path::Path,
) -> (
WorkspaceApi,
Arc<DeterministicExecutionBackend>,
String,
crate::store::WorkerCreateReservation,
) {
let (api, backend) = test_api_with_recording_backend(root).await;
let subject = open_subjektiv_store(&api)
.unwrap()
.create_subject(crate::subjektiv::SubjectRole::new("recovery-test").unwrap())
.unwrap();
let key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
let memory = api
.config_store
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
.unwrap();
let old = api
.config_store
.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID,
"manual:failed-subject",
&"f".repeat(64),
Some(&key),
&memory,
)
.unwrap();
api.config_store
.fail_worker_create_reservation(
TEST_WORKSPACE_ID,
EMBEDDED_RUNTIME_ID,
old.worker_id,
&old.create_fingerprint,
)
.unwrap();
(api, backend, subject.id, old)
}
fn recovery_spawn_request(subject: &str) -> WorkerSpawnRequest {
serde_json::from_value(json!({
"intent": WorkerSpawnIntent::WorkspaceCompanion,
"acceptance": WorkerSpawnAcceptanceRequirement::RunAccepted { expected_segments: 0 },
"profile": worker_runtime::catalog::ProfileSelector::Builtin("builtin:companion".into()),
"singleton_key": crate::subjektiv::subject_worker_singleton_key(subject).unwrap(),
}))
.unwrap()
}
#[tokio::test]
async fn unpublished_create_recovery_retries_same_fence_then_creates_new_worker() {
let root = tempfile::tempdir().unwrap();
let (api, backend, subject, old) = unpublished_recovery_fixture(root.path()).await;
let old_worker = RuntimeWorkerRef::new(EMBEDDED_RUNTIME_ID, old.worker_id.to_string());
let lease = old.singleton.as_ref().unwrap();
let late = crate::subjektiv::SubjectSessionAttribution::new(
&subject,
EMBEDDED_RUNTIME_ID,
old.worker_id.to_string(),
session_store::new_session_id().to_string(),
)
.unwrap();
let late_session = late.session_id.clone();
// Models an already-authorized callback arriving while Runtime fencing is
// in flight. The Host intent must reject it BEFORE attribution insertion.
let callback_api = api.clone();
*backend.recovery_hook.lock().unwrap() = Some(Box::new(move || {
assert!(subjektiv_journal::bind_host_session(&callback_api, late).is_err());
}));
*backend.recovery_failure.lock().unwrap() = Some("unconfirmed execution absence".into());
assert!(
api.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject))
.is_err()
);
assert!(
api.config_store
.subject_release_pending(TEST_WORKSPACE_ID, &old_worker)
.unwrap()
);
assert_eq!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap(),
Some(lease.clone())
);
let store = open_subjektiv_store(&api).unwrap();
assert!(store.session_attribution(&late_session).unwrap().is_none());
assert!(
store
.journal_body_is_unpublished(&subject, EMBEDDED_RUNTIME_ID, &old_worker.worker_id)
.unwrap()
);
let replacement = api
.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject))
.unwrap();
assert_eq!(replacement.state, InternalWorkerOperationState::Accepted);
let replacement = replacement.worker.unwrap().worker;
assert_ne!(replacement, old_worker);
let current = api
.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap()
.unwrap();
assert_eq!(current.worker, replacement);
assert_eq!(current.generation, lease.generation + 1);
let requests = backend.recovery_requests.lock().unwrap();
assert_eq!(requests.len(), 2);
assert_eq!(
requests[0], requests[1],
"unknown result must reuse exact recovery identity"
);
// A stale completion cannot release the replacement generation.
api.config_store
.complete_subject_create_recovery(
TEST_WORKSPACE_ID,
lease,
&requests[0].operation_id,
&old.create_fingerprint,
)
.unwrap();
assert_eq!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap(),
Some(current)
);
}
#[tokio::test]
async fn unpublished_create_recovery_refuses_any_attribution_or_orphan_writer() {
for orphan_writer in [false, true] {
let root = tempfile::tempdir().unwrap();
let (api, backend, subject, old) = unpublished_recovery_fixture(root.path()).await;
let store = open_subjektiv_store(&api).unwrap();
if orphan_writer {
store
.claim_journal_writer(&subject, EMBEDDED_RUNTIME_ID, &old.worker_id.to_string())
.unwrap();
} else {
store
.record_session_attribution(
crate::subjektiv::SubjectSessionAttribution::new(
&subject,
EMBEDDED_RUNTIME_ID,
old.worker_id.to_string(),
session_store::new_session_id().to_string(),
)
.unwrap(),
)
.unwrap();
}
assert!(
api.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject))
.is_err()
);
assert!(backend.recovery_requests.lock().unwrap().is_empty());
assert_eq!(
api.config_store
.current_worker_singleton_owner(
TEST_WORKSPACE_ID,
&old.singleton.as_ref().unwrap().key
)
.unwrap(),
old.singleton
);
}
}
#[tokio::test]
async fn unpublished_create_recovery_resumes_after_runtime_proof_before_host_commit() {
let root = tempfile::tempdir().unwrap();
let (api, backend, subject, old) = unpublished_recovery_fixture(root.path()).await;
let lease = old.singleton.as_ref().unwrap();
let op = format!(
"subject-create-recovery:{}:{}",
old.worker_id, lease.generation
);
let fingerprint = api
.config_store
.prepare_subject_create_recovery(TEST_WORKSPACE_ID, lease, &op)
.unwrap();
let request = runtime_api::SubjectCreateRecoveryRequest {
workspace_id: TEST_WORKSPACE_ID.into(),
worker_id: old.worker_id,
create_fingerprint: fingerprint,
operation_id: op,
};
api.runtime
.recover_subject_create(EMBEDDED_RUNTIME_ID, request)
.unwrap();
// Simulate loss of the receipt before Host completion. The Host must not
// infer completion from missing catalog state, and can retry the exact op.
assert_eq!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap(),
old.singleton
);
subjektiv_journal::recover_failed_subject_create(&api, &subject).unwrap();
assert!(
api.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key)
.unwrap()
.is_none()
);
assert_eq!(backend.recovery_requests.lock().unwrap().len(), 2);
}
#[tokio::test]
async fn failed_subject_spawn_can_recover_on_next_explicit_create() {
let root = tempfile::tempdir().unwrap();
let (api, backend) = test_api_with_recording_backend(root.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("failed-spawn").unwrap())
.unwrap();
let key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
*backend.spawn_failure.lock().unwrap() =
Some("Session registration failed before publication".into());
let rejected = api
.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject.id))
.unwrap();
assert_eq!(rejected.state, InternalWorkerOperationState::Rejected);
let old = api
.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
.unwrap()
.unwrap();
assert!(
api.config_store
.subject_create_recovery_candidate(TEST_WORKSPACE_ID, &old)
.unwrap()
.is_some()
);
assert!(
backend.recovery_requests.lock().unwrap().is_empty(),
"failure alone must not release occupancy"
);
let result = api
.spawn_workspace_worker(EMBEDDED_RUNTIME_ID, recovery_spawn_request(&subject.id))
.unwrap();
assert_eq!(result.state, InternalWorkerOperationState::Accepted);
let current = api
.config_store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
.unwrap()
.unwrap();
assert_ne!(current.worker, old.worker);
assert_eq!(current.generation, old.generation + 1);
assert_eq!(backend.recovery_requests.lock().unwrap().len(), 1);
}
crates/workspace-server/src/store.rs
+591 -6
View File
@@ -119,7 +119,7 @@ pub use ticket::{
};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 101;
const LATEST_SCHEMA_VERSION: i64 = 102;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -432,6 +432,11 @@ const MIGRATIONS: &[Migration] = &[
name: "Durable explicit Subject body release",
apply: migrate_subject_release_v100_to_v101,
},
Migration {
version: 102,
name: "Durable failed Subject creation recovery",
apply: migrate_subject_create_recovery_v101_to_v102,
},
];
#[derive(Clone, Copy)]
@@ -2997,7 +3002,7 @@ impl SqliteWorkspaceStore {
.query_row(
"SELECT allocation_key, request_fingerprint, create_fingerprint, \
memory_language, singleton_key, \
singleton_generation \
singleton_generation, state \
FROM worker_create_reservations \
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3",
params![
@@ -3013,11 +3018,21 @@ impl SqliteWorkspaceStore {
row.get::<_, Option<String>>(3)?,
row.get::<_, Option<String>>(4)?,
row.get::<_, Option<i64>>(5)?,
row.get::<_, String>(6)?,
))
},
)
.optional()?;
if let Some((stored_allocation_key, stored_request_fingerprint, create_fingerprint, language, stored_singleton_key, stored_generation)) = stored
// Subject occupancy survives removal until an explicit release
// proof. It must not resurrect the removed creation reservation:
// Runtime callbacks require a live reservation or catalog member.
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
return Err(Error::RepositoryConflict(format!(
"worker_singleton_release_required: Worker {}:{} has a terminal create reservation but still owns the Subject; complete Subject release before creating a replacement",
owner.worker.runtime_id, owner.worker.worker_id
)));
}
if let Some((stored_allocation_key, stored_request_fingerprint, create_fingerprint, language, stored_singleton_key, stored_generation, _)) = stored
&& allocation_key.starts_with("manual:")
&& stored_allocation_key.starts_with("manual:")
&& owner.worker.runtime_id == runtime_id
@@ -3199,6 +3214,99 @@ impl SqliteWorkspaceStore {
})
}
/// Eligibility is Backend evidence only. The Host must separately prove that Subjektiv
/// never published this body and obtain the Runtime's exact create-fence receipt.
pub(crate) fn subject_create_recovery_candidate(
&self,
workspace: &str,
lease: &WorkerSingletonLease,
) -> Result<Option<String>> {
self.with_conn(|conn| subject_create_recovery_candidate(conn, workspace, lease))
}
pub(crate) fn prepare_subject_create_recovery(
&self,
workspace: &str,
lease: &WorkerSingletonLease,
operation_id: &str,
) -> Result<String> {
if operation_id.trim().is_empty() || operation_id.len() > 512 {
return Err(Error::InvalidInput(
"invalid Subject create recovery operation".into(),
));
}
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let fingerprint = subject_create_recovery_candidate(&tx, workspace, lease)?
.ok_or_else(|| Error::RepositoryConflict("Subject create recovery requires the current removed reservation without a registry row".into()))?;
let saved: Option<(String, String, String, u64, String)> = tx.query_row(
"SELECT runtime_id,worker_id,singleton_key,singleton_generation,create_fingerprint FROM subject_create_recoveries WHERE workspace_id=?1 AND operation_id=?2",
params![workspace, operation_id],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)),
).optional()?;
if let Some(saved) = saved {
if saved != (lease.worker.runtime_id.clone(), lease.worker.worker_id.clone(), lease.key.clone(), lease.generation, fingerprint.clone()) {
return Err(Error::RepositoryConflict("Subject create recovery operation was used with different input".into()));
}
return Ok(fingerprint);
}
let busy: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_create_recoveries WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5)",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, lease.generation as i64],
|row| row.get(0),
)?;
if busy {
return Err(Error::RepositoryConflict("Subject create recovery already has a durable operation".into()));
}
tx.execute(
"INSERT INTO subject_create_recoveries(workspace_id,runtime_id,worker_id,singleton_key,singleton_generation,operation_id,create_fingerprint) VALUES(?1,?2,?3,?4,?5,?6,?7)",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, lease.generation as i64, operation_id, fingerprint],
)?;
tx.commit()?;
Ok(fingerprint)
})
}
/// Called only after the Host verifies the Runtime's typed receipt for this exact create.
/// The completed bit records that proof; it neither creates a writer binding nor rewrites
/// the reservation. An exact completed replay is a receipt, not authority over a successor.
pub(crate) fn complete_subject_create_recovery(
&self,
workspace: &str,
lease: &WorkerSingletonLease,
operation_id: &str,
create_fingerprint: &str,
) -> Result<()> {
let generation = i64::try_from(lease.generation)
.map_err(|_| Error::InvalidInput("invalid singleton generation".into()))?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let completed: Option<bool> = tx.query_row(
"SELECT completed FROM subject_create_recoveries WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND operation_id=?6 AND create_fingerprint=?7",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, generation, operation_id, create_fingerprint],
|row| row.get(0),
).optional()?;
match completed {
Some(true) => return Ok(()),
Some(false) => {},
None => return Err(Error::RepositoryConflict("Subject create recovery intent absent or mismatched".into())),
}
let current = subject_create_recovery_candidate(&tx, workspace, lease)?;
if current.as_deref() != Some(create_fingerprint) {
return Err(Error::RepositoryConflict("Subject create recovery reservation or singleton ownership changed".into()));
}
let changed = tx.execute(
"UPDATE subject_create_recoveries SET completed=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND operation_id=?6 AND create_fingerprint=?7 AND completed=0",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, generation, operation_id, create_fingerprint],
)?;
if changed != 1 {
return Err(Error::RepositoryConflict("Subject create recovery completion lost its exact intent".into()));
}
tx.commit()?;
Ok(())
})
}
pub(crate) fn retained_subject_release_operation(
&self,
workspace: &str,
@@ -3213,7 +3321,7 @@ impl SqliteWorkspaceStore {
workspace: &str,
worker: &RuntimeWorkerRef,
) -> Result<bool> {
self.with_conn(|c| Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND cancelled=0 AND completed=0)",params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0))?))
self.with_conn(|c| Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND cancelled=0 AND completed=0) OR EXISTS(SELECT 1 FROM subject_create_recoveries WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND completed=0)",params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0))?))
}
pub(crate) fn require_subject_release_intent(
@@ -15012,6 +15120,34 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
Ok(())
}
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 101 {
return Err(Error::Store(
"Subject create recovery migration requires schema 101".into(),
));
}
let tx = conn.unchecked_transaction()?;
tx.execute_batch("CREATE TABLE subject_create_recoveries (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
singleton_key TEXT NOT NULL,
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
operation_id TEXT NOT NULL CHECK (length(operation_id) > 0),
create_fingerprint TEXT NOT NULL,
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
PRIMARY KEY (workspace_id, operation_id),
UNIQUE (workspace_id, runtime_id, worker_id, singleton_key, singleton_generation),
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
);")?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(102,?1)",
["Durable failed Subject creation recovery"],
)?;
tx.commit()?;
Ok(())
}
fn migrate_subject_release_v100_to_v101(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 100 {
return Err(Error::Store(
@@ -15702,6 +15838,33 @@ pub(crate) fn validate_worker_singleton_key(key: &str) -> Result<()> {
Ok(())
}
fn subject_create_recovery_candidate(
conn: &Connection,
workspace: &str,
lease: &WorkerSingletonLease,
) -> Result<Option<String>> {
if !lease
.key
.starts_with(crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX)
{
return Err(Error::InvalidInput("not a Subject singleton".into()));
}
let generation = i64::try_from(lease.generation)
.map_err(|_| Error::InvalidInput("invalid singleton generation".into()))?;
Ok(conn.query_row(
"SELECT r.create_fingerprint FROM worker_create_reservations r
JOIN worker_singleton_owners o ON o.workspace_id=r.workspace_id
AND o.runtime_id=r.runtime_id AND o.worker_id=r.worker_id
AND o.singleton_key=r.singleton_key AND o.generation=r.singleton_generation
WHERE r.workspace_id=?1 AND r.runtime_id=?2 AND r.worker_id=?3
AND r.singleton_key=?4 AND r.singleton_generation=?5 AND r.state='removed'
AND NOT EXISTS(SELECT 1 FROM worker_registry w
WHERE w.workspace_id=r.workspace_id AND w.runtime_id=r.runtime_id AND w.worker_id=r.worker_id)",
params![workspace, lease.worker.runtime_id, lease.worker.worker_id, lease.key, generation],
|row| row.get(0),
).optional()?)
}
fn current_worker_singleton_owner(
conn: &Connection,
workspace_id: &str,
@@ -15712,6 +15875,7 @@ fn current_worker_singleton_owner(
"SELECT owner.runtime_id, owner.worker_id, owner.generation, \
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
THEN 'created' ELSE reservation.state END \
FROM worker_singleton_owners owner \
LEFT JOIN worker_create_reservations reservation \
@@ -18739,6 +18903,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
WorkspaceSchemaMigrationStep {version:99, name:"Explicit hosting administrators and self-registration policy".into()},
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
]
);
@@ -18850,6 +19015,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(99, "Explicit hosting administrators and self-registration policy".to_string()),
(100, "Provisional users and scoped passkey activation".to_string()),
(101, "Durable explicit Subject body release".to_string()),
(102, "Durable failed Subject creation recovery".to_string()),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -20907,6 +21073,47 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
conflict,
Error::RepositoryConflict(message) if message.contains("worker_singleton_owned")
));
reopened
.fail_worker_create_reservation(
"workspace-a",
"runtime-a",
reserved.worker_id,
&reserved.create_fingerprint,
)
.unwrap();
drop(reopened);
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
let conflict = reopened
.reserve_worker_create(
"workspace-a",
"runtime-a",
"manual:retry-after-terminal-failure",
&stable_fingerprint,
Some("subjektiv:restart-retry"),
&memory,
)
.unwrap_err();
assert!(matches!(conflict, Error::RepositoryConflict(message)
if message.contains("worker_singleton_release_required")));
let worker = RuntimeWorkerRef::new("runtime-a", reserved.worker_id.to_string());
assert!(
!reopened
.has_active_worker_create_reservation("workspace-a", &worker)
.unwrap()
);
assert!(
reopened
.worker_resource_key("workspace-a", &worker)
.unwrap()
.is_none()
);
assert_eq!(
reopened
.current_worker_singleton_owner("workspace-a", "subjektiv:restart-retry")
.unwrap(),
reserved.singleton
);
}
#[tokio::test]
@@ -21036,6 +21243,383 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
);
}
#[test]
fn subject_create_recovery_migration_is_atomic_and_matches_fresh_schema() {
let dir = tempfile::tempdir().unwrap();
let database = dir.path().join("server.db");
prepare_retained_schema(&database, 101);
let conn = Connection::open(&database).unwrap();
configure_sqlite(&conn).unwrap();
conn.execute_batch("CREATE TRIGGER interrupt_create_recovery BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=102 BEGIN SELECT RAISE(ABORT,'injected recovery migration interruption'); END;").unwrap();
assert!(migrate_subject_create_recovery_v101_to_v102(&conn).is_err());
assert_eq!(current_schema_version(&conn).unwrap(), 101);
assert!(!table_exists(&conn, "subject_create_recoveries").unwrap());
conn.execute_batch("DROP TRIGGER interrupt_create_recovery")
.unwrap();
migrate_subject_create_recovery_v101_to_v102(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 102);
assert!(migrate_subject_create_recovery_v101_to_v102(&conn).is_err());
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
let schema = |c: &Connection| -> Result<String> {
Ok(c.query_row(
"SELECT sql FROM sqlite_schema WHERE name='subject_create_recoveries'",
[],
|r| r.get::<_, String>(0),
)?
.split_whitespace()
.collect::<Vec<_>>()
.join(" "))
};
assert_eq!(schema(&conn).unwrap(), fresh.with_conn(schema).unwrap());
assert_eq!(
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r
.get::<_, i64>(
0
))
.unwrap(),
0
);
drop(conn);
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
reopened
.with_conn(|c| verify_schema_history(c, 102))
.unwrap();
}
#[tokio::test]
async fn subject_create_recovery_is_durable_exact_and_cannot_release_a_new_generation() {
for baseline in [None, Some(52), Some(101)] {
let dir = tempfile::tempdir().unwrap();
let database = dir.path().join("server.db");
if let Some(baseline) = baseline {
prepare_retained_schema(&database, baseline);
}
let store = SqliteWorkspaceStore::open_api_fixture(&database).unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace-a".into(),
owner_account_id: "owner-account".into(),
display_name: "Workspace A".into(),
state: "active".into(),
created_at: "2026-10-01T00:00:00Z".into(),
updated_at: "2026-10-01T00:00:00Z".into(),
})
.await
.unwrap();
let memory = store.get_workspace_memory_settings("workspace-a").unwrap();
let key = "subjektiv:failed-create";
let first = store
.reserve_worker_create(
"workspace-a",
"runtime-a",
"manual:first",
"sha256:first",
Some(key),
&memory,
)
.unwrap();
let lease = first.singleton.unwrap();
let fingerprint = first.create_fingerprint;
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &lease)
.unwrap(),
None
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.is_err()
);
store
.fail_worker_create_reservation(
"workspace-a",
"runtime-a",
first.worker_id,
&fingerprint,
)
.unwrap();
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &lease)
.unwrap(),
Some(fingerprint.clone())
);
assert!(
store
.prepare_subject_create_recovery("workspace-b", &lease, "recover-first")
.is_err()
);
for wrong in [
WorkerSingletonLease {
generation: lease.generation + 1,
..lease.clone()
},
WorkerSingletonLease {
key: "subjektiv:other".into(),
..lease.clone()
},
WorkerSingletonLease {
worker: RuntimeWorkerRef::new("runtime-b", &lease.worker.worker_id),
..lease.clone()
},
WorkerSingletonLease {
worker: RuntimeWorkerRef::new("runtime-a", "other-worker"),
..lease.clone()
},
] {
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &wrong)
.unwrap(),
None
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &wrong, "recover-first")
.is_err()
);
}
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
assert_eq!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.unwrap(),
fingerprint
);
assert!(
store
.subject_release_pending("workspace-a", &lease.worker)
.unwrap()
);
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
Some(lease.clone())
);
assert!(
store
.reserve_worker_create(
"workspace-a",
"runtime-a",
"manual:retry",
"sha256:first",
Some(key),
&memory
)
.is_err()
);
drop(store);
let store = SqliteWorkspaceStore::open_for_test(&database).unwrap();
assert_eq!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.unwrap(),
fingerprint
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "different-operation")
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
"different-fingerprint"
)
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"different-operation",
&fingerprint
)
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-b",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
// A registry row appearing after prepare must block both replay and completion.
store.with_conn(|c| {
c.execute("INSERT INTO worker_registry(workspace_id,runtime_id,worker_id,display_name,retention_state,created_at,updated_at) VALUES('workspace-a',?1,?2,'body','normal','now','now')", params![lease.worker.runtime_id, lease.worker.worker_id])?;
Ok(())
}).unwrap();
assert_eq!(
store
.subject_create_recovery_candidate("workspace-a", &lease)
.unwrap(),
None
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
store.with_conn(|c| {
c.execute("DELETE FROM worker_registry WHERE workspace_id='workspace-a' AND runtime_id=?1 AND worker_id=?2", params![lease.worker.runtime_id, lease.worker.worker_id])?;
// Exercise a current-generation change while the operation is incomplete.
c.execute("UPDATE worker_singleton_owners SET generation=generation+1 WHERE workspace_id='workspace-a' AND singleton_key=?1", [key])?;
Ok(())
}).unwrap();
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint
)
.is_err()
);
assert!(
store
.subject_release_pending("workspace-a", &lease.worker)
.unwrap()
);
store.with_conn(|c| {
c.execute("UPDATE worker_singleton_owners SET generation=?2 WHERE workspace_id='workspace-a' AND singleton_key=?1", params![key, lease.generation as i64])?;
Ok(())
}).unwrap();
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint,
)
.unwrap();
drop(store);
let store = SqliteWorkspaceStore::open_for_test(&database).unwrap();
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint,
)
.unwrap();
assert!(
!store
.subject_release_pending("workspace-a", &lease.worker)
.unwrap()
);
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
None
);
let next = store
.reserve_worker_create(
"workspace-a",
"runtime-b",
"manual:next",
"sha256:next",
Some(key),
&memory,
)
.unwrap();
let next_lease = next.singleton.unwrap();
assert_eq!(next_lease.generation, lease.generation + 1);
store
.complete_subject_create_recovery(
"workspace-a",
&lease,
"recover-first",
&fingerprint,
)
.unwrap();
assert!(
store
.prepare_subject_create_recovery("workspace-a", &lease, "recover-first")
.is_err()
);
assert!(
store
.complete_subject_create_recovery(
"workspace-a",
&next_lease,
"recover-first",
&fingerprint
)
.is_err()
);
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
Some(next_lease.clone())
);
// The old completed proof cannot release the next removed generation.
store
.fail_worker_create_reservation(
"workspace-a",
"runtime-b",
next.worker_id,
&next.create_fingerprint,
)
.unwrap();
assert_eq!(
store
.current_worker_singleton_owner("workspace-a", key)
.unwrap(),
Some(next_lease.clone())
);
assert!(
store
.prepare_subject_create_recovery("workspace-a", &next_lease, "recover-first")
.is_err()
);
assert_eq!(
store
.prepare_subject_create_recovery("workspace-a", &next_lease, "recover-next")
.unwrap(),
next.create_fingerprint
);
store.with_conn(|c| {
assert_eq!(c.query_row("SELECT count(*) FROM subject_worker_releases", [], |r| r.get::<_, i64>(0))?, 0);
assert_eq!(c.query_row("SELECT state FROM worker_create_reservations WHERE workspace_id='workspace-a' AND worker_id=?1", [&lease.worker.worker_id], |r| r.get::<_, String>(0))?, "removed");
assert!(c.execute("UPDATE subject_create_recoveries SET completed=2", []).is_err());
assert!(c.execute("UPDATE subject_create_recoveries SET singleton_generation=0", []).is_err());
assert_eq!(c.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r.get::<_, i64>(0))?, 0);
Ok(())
}).unwrap();
}
}
#[test]
fn subject_release_migration_rolls_back_schema_and_marker_then_reopens() {
let dir = tempfile::tempdir().unwrap();
@@ -21056,7 +21640,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
drop(conn);
let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap();
reopened
.with_conn(|conn| verify_schema_history(conn, 101))
.with_conn(|conn| verify_schema_history(conn, LATEST_SCHEMA_VERSION))
.unwrap();
}
@@ -21636,7 +22220,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(98, "Assignment-independent implementation reports".into()),
(99, "Explicit hosting administrators and self-registration policy".into()),
(100, "Provisional users and scoped passkey activation".into()),
(101, "Durable explicit Subject body release".into())]);
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into())]);
for (table, rows) in unchanged.iter().zip(before.iter()) {
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
}
crates/workspace-server/src/workspace_deletion.rs
+1
View File
@@ -95,6 +95,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"workdir_removal_operations",
"worker_control_grants",
"subject_worker_releases",
"subject_create_recoveries",
"worker_create_reservations",
"worker_diagnostics_archives",
"worker_mutation_source_proof_jtis",