diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index 940506ca7..ae155b008 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -432,16 +432,23 @@ mod client { use super::*; use crate::{Workdir, WorkdirSession}; - /// Provides a fresh bearer token for each Runtime request. Backend - /// implementations can mint short-lived capability tokens without making a - /// Worker-bound session expire with the token used to open it. + /// An owned, Send authorization future that can offload synchronous authority + /// work without borrowing request data or the authorization provider. + pub type WorkdirHttpAuthorizationFuture = std::pin::Pin< + Box> + Send + 'static>, + >; + + /// Provides a fresh bearer token asynchronously for each Runtime request. + /// Backend implementations can mint short-lived capability tokens without + /// making a Worker-bound session expire with the token used to open it. + /// Blocking DB/signing work must be offloaded by the implementation. pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync { fn bearer_token( &self, - method: &str, - path_and_query: &str, - body: &[u8], - ) -> Result; + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture; } struct FixedBearerToken(Arc); @@ -455,11 +462,12 @@ mod client { impl WorkdirHttpAuthorization for FixedBearerToken { fn bearer_token( &self, - _method: &str, - _path_and_query: &str, - _body: &[u8], - ) -> Result { - Ok(self.0.to_string()) + _method: String, + _path_and_query: String, + _body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let token = self.0.clone(); + Box::pin(async move { Ok(token.to_string()) }) } } @@ -509,7 +517,9 @@ mod client { )?; let body = serde_json::to_vec(&request) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = authorization.bearer_token("POST", url.path(), &body)?; + let token = authorization + .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) + .await?; let response = client .post(url) .bearer_auth(token) @@ -560,7 +570,10 @@ mod client { let operation = WorkdirSessionOperationRequest { operation }; let body = serde_json::to_vec(&operation) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = self.authorization.bearer_token("POST", url.path(), &body)?; + let token = self + .authorization + .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) + .await?; let response = self .client .post(url) @@ -864,7 +877,10 @@ mod client { &self.base_url, &["v1", "workdir-sessions", self.session_id.as_str()], )?; - let token = self.authorization.bearer_token("DELETE", url.path(), &[])?; + let token = self + .authorization + .bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new()) + .await?; let response = self .client .delete(url) @@ -918,11 +934,124 @@ mod client { WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string()) } + #[cfg(test)] + mod authorization_tests { + use super::*; + use std::sync::Mutex; + + #[derive(Debug, Default)] + struct RejectAuthorization { + requests: Arc)>>>, + } + + impl WorkdirHttpAuthorization for RejectAuthorization { + fn bearer_token( + &self, + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let requests = self.requests.clone(); + Box::pin(async move { + tokio::task::yield_now().await; + let mut requests = requests.lock().unwrap(); + requests.push((method, path_and_query, body)); + Err(WorkdirError::Unavailable(format!( + "authorization rejected request {}", + requests.len() + ))) + }) + } + } + + #[tokio::test] + async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let base_url: Url = format!("http://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(); + let client = Client::builder().no_proxy().build().unwrap(); + let authorization = Arc::new(RejectAuthorization::default()); + let workdir = Workdir::new("w"); + let open_request = OpenWorkdirSessionRequest { + owner_worker_id: Some("worker".to_string()), + }; + let error = RemoteWorkdirSession::open_with_authorization( + client.clone(), + base_url.clone(), + authorization.clone(), + workdir.id().clone(), + open_request.clone(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 1" + )); + + // Seed an already-open session to exercise the public operation and + // close boundaries independently of the rejected open request. + let remote = RemoteWorkdirSession { + client, + base_url, + authorization: authorization.clone(), + workdir, + session_id: WorkdirSessionId::new("s").unwrap(), + capabilities: WorkdirSessionCapabilities::EMPTY, + closed: AtomicBool::new(false), + }; + let stat = StatRequest { + path: crate::WorkdirPath::new("file").unwrap(), + }; + let error = remote.stat(stat.clone()).await.unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 2" + )); + let error = remote.close().await.unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 3" + )); + assert_eq!( + *authorization.requests.lock().unwrap(), + vec![ + ( + "POST".to_string(), + "/v1/working-directories/w/sessions".to_string(), + serde_json::to_vec(&open_request).unwrap(), + ), + ( + "POST".to_string(), + "/v1/workdir-sessions/s/operations".to_string(), + serde_json::to_vec(&WorkdirSessionOperationRequest { + operation: WorkdirSessionOperation::Stat(stat), + }) + .unwrap(), + ), + ( + "DELETE".to_string(), + "/v1/workdir-sessions/s".to_string(), + Vec::new(), + ), + ] + ); + assert_eq!( + listener.accept().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); + } + } + pub use self::RemoteWorkdirSession as ClientSession; } #[cfg(feature = "http-client")] -pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization}; +pub use client::{ + ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture, +}; #[cfg(test)] mod tests { diff --git a/crates/workspace-server/src/assignment_delivery_tests.rs b/crates/workspace-server/src/assignment_delivery_tests.rs index b8702edb7..c8e2cb7f6 100644 --- a/crates/workspace-server/src/assignment_delivery_tests.rs +++ b/crates/workspace-server/src/assignment_delivery_tests.rs @@ -147,7 +147,7 @@ async fn assignment_delivery_response_loss_returns_immutable_payload_after_progr .update_ticket_assignment_tracking("w", "tracking", &update, &source) .unwrap(); drop(store); - let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); assert_eq!( store .list_pending_ticket_assignment_deliveries("w", 100) @@ -208,7 +208,7 @@ async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt() let path = directory.path().join("workspace.db"); let input = pending.input_event_id.clone(); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open(path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(path).unwrap(); barrier.wait(); store .reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source()) @@ -568,7 +568,7 @@ async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_co "original rendered prompt" ); drop(store); - let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); assert_eq!( store .prepare_ticket_assignment_delivery_content( @@ -634,7 +634,7 @@ fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_l let operations = stored_fixture_rows(&conn, "ticket_assignment_operations"); let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history"); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let pending = store .list_pending_ticket_assignment_deliveries("w", 100) .unwrap(); @@ -684,7 +684,7 @@ async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once let barrier = barrier.clone(); let path = directory.path().join("workspace.db"); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open(path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(path).unwrap(); barrier.wait(); delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source()) .unwrap() @@ -716,8 +716,8 @@ fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() { let conn = tracking_schema91_fixture(&path); migrate_ticket_tracking_v91_to_v92(&conn).unwrap(); drop(conn); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let objects = |store: &SqliteWorkspaceStore| { store.with_conn(|conn| { let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?; @@ -741,7 +741,7 @@ async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() { .unwrap(); drop(store); assert!( - matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) + matches!(SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) ); } diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 513daef26..5255267c7 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -78,7 +78,7 @@ async fn tracking_store_every_replay_fences_current_singleton_and_authenticated_ ); // Model the authority change after authenticated source capture, before // BEGIN IMMEDIATE on a second connection. Receipt presence cannot bypass it. - let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); second.with_conn_mut(|conn| { let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; match change { @@ -233,7 +233,7 @@ async fn tracking_store_same_principal_reissue_and_exact_retries_keep_original_r .is_err() ); drop(store); - let reopened = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); assert_eq!( reopened .get_ticket_assignment_tracking("w", "tracking", "a") @@ -441,7 +441,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .collect::>(), vec![92, 93] ); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, 93); @@ -525,7 +525,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .list_ticket_assignment_tracking_history("w", "pending", "a-pending") .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( reopened .list_ticket_assignment_tracking_history("w", "pending", "a-pending") @@ -550,7 +550,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first let releases_before = stored_fixture_rows(&conn, "ticket_assignment_work_releases"); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "w").unwrap(); let reopen = || { let observed = backend.show("historical-done".into()).unwrap(); @@ -645,7 +645,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first ); // A second observer holds the same Assign baseline. Only the first // explicit writer may consume it; BEGIN IMMEDIATE serializes the CAS. - let second = SqliteWorkspaceStore::open(&path).unwrap(); + let second = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( second .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") @@ -740,7 +740,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first }).unwrap(); drop(second); drop(store); - let reopened_store = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened_store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( reopened_store .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") @@ -796,7 +796,7 @@ fn tracking_store_schema91_missing_assign_audit_remains_unknown_without_cas_base ) .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let unknown = store .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") .unwrap() @@ -1068,7 +1068,7 @@ async fn orchestrator_assignment_all_runtime_retries_fence_singleton_and_binding &source, ) .unwrap(); - let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); second.with_conn_mut(|conn| {let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;match change { "replace" => {tx.execute("UPDATE worker_singleton_owners SET worker_id='replacement',generation=2 WHERE workspace_id='w'",[])?;}, "revoke" => {tx.execute("UPDATE workspace_runtime_bindings SET revoked_at='revoked' WHERE workspace_id='w'",[])?;}, diff --git a/crates/workspace-server/src/auth.rs b/crates/workspace-server/src/auth.rs index aa202f2de..b5badef0d 100644 --- a/crates/workspace-server/src/auth.rs +++ b/crates/workspace-server/src/auth.rs @@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option { }) } -pub async fn resolve_request_actor( - store: &S, +pub async fn resolve_request_actor( + store: std::sync::Arc, headers: &HeaderMap, cookie_name: &str, ) -> Result> { - if let Some(token) = parse_bearer(headers) { + let bearer = parse_bearer(headers); + let session = parse_cookie(headers, cookie_name); + if bearer.is_none() && session.is_none() { + return Ok(None); + } + let executor = store.db_executor(); + executor + .run(move || resolve_request_actor_sync(store.as_ref(), bearer, session)) + .await +} + +// One owned synchronous stage includes token usage marking and the user read. +// Do not call async store wrappers here: they use the same executor themselves. +fn resolve_request_actor_sync( + store: &dyn ControlPlaneStore, + bearer: Option, + session: Option, +) -> Result> { + if let Some(token) = bearer { let token_hash = token_hash(&token); if let Some(api_token) = store.resolve_api_token(&token_hash)? { if api_token @@ -109,7 +127,7 @@ pub async fn resolve_request_actor( } } - if let Some(session_token) = parse_cookie(headers, cookie_name) { + if let Some(session_token) = session { if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? { if is_expired(&session.expires_at) { return Ok(None); @@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error { #[cfg(test)] mod tests { use super::*; + use crate::store::{ + AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord, + }; + use std::sync::Arc; + + fn seeded_store() -> Arc { + let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::in_memory_for_test() + .unwrap() + .with_db_executor(executor), + ); + store + .upsert_account(&AccountRecord { + account_id: "account-auth".into(), + kind: "user".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .upsert_user(&UserRecord { + user_id: "user-auth".into(), + account_id: "account-auth".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .create_browser_session(&BrowserSessionRecord { + session_id: "session-auth".into(), + token_hash: token_hash("session-secret"), + user_id: "user-auth".into(), + created_at: "1".into(), + expires_at: "9999-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + store + } + + #[tokio::test] + async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() { + let store = seeded_store(); + let hash = token_hash("api-secret"); + store + .create_api_token(&ApiTokenRecord { + token_id: "token-auth".into(), + token_hash: hash.clone(), + user_id: "user-auth".into(), + label: "Auth".into(), + created_at: "1".into(), + expires_at: None, + revoked_at: None, + last_used_at: None, + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap()); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let actor = resolve_request_actor(store.clone(), &headers, "session") + .await + .unwrap() + .unwrap(); + assert_eq!(actor.user_id, "user-auth"); + assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken); + assert!( + store + .resolve_api_token(&hash) + .unwrap() + .unwrap() + .last_used_at + .is_some() + ); + } + + #[tokio::test] + async fn expired_api_token_does_not_fall_back_to_valid_browser_session() { + let store = seeded_store(); + let hash = token_hash("expired-secret"); + store + .create_api_token(&ApiTokenRecord { + token_id: "token-auth".into(), + token_hash: hash.clone(), + user_id: "user-auth".into(), + label: "Auth".into(), + created_at: "1".into(), + expires_at: Some("2000-01-01T00:00:00Z".into()), + revoked_at: None, + last_used_at: None, + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + "Bearer expired-secret".parse().unwrap(), + ); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + assert!( + resolve_request_actor(store.clone(), &headers, "session") + .await + .unwrap() + .is_none() + ); + assert!( + store + .resolve_api_token(&hash) + .unwrap() + .unwrap() + .last_used_at + .is_none() + ); + } + + #[tokio::test] + async fn unknown_bearer_can_fall_back_to_valid_browser_session() { + let store = seeded_store(); + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + "Bearer unknown-secret".parse().unwrap(), + ); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let actor = resolve_request_actor(store, &headers, "session") + .await + .unwrap() + .unwrap(); + assert_eq!(actor.user_id, "user-auth"); + assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession); + } + + #[tokio::test] + async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() { + let store = seeded_store(); + let executor = store.db_executor(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let blocker_executor = executor.clone(); + let blocker = tokio::spawn(async move { + blocker_executor + .run(move || -> Result<()> { + let _ = started.send(()); + release_rx.recv().unwrap(); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(1), started_rx) + .await + .unwrap() + .unwrap(); + let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) })); + assert!(futures::poll!(queued.as_mut()).is_pending()); + let mut headers = HeaderMap::new(); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + resolve_request_actor(store, &headers, "session"), + ) + .await; + release.send(()).unwrap(); + blocker.await.unwrap().unwrap(); + queued.await.unwrap(); + assert!(matches!( + result.expect("capacity rejection must not wait for queued SQL"), + Err(Error::DbCapacityExceeded) + )); + } #[test] fn user_code_uses_the_uuid_random_tail() { diff --git a/crates/workspace-server/src/authority.rs b/crates/workspace-server/src/authority.rs index 1203f2503..f52ca0f20 100644 --- a/crates/workspace-server/src/authority.rs +++ b/crates/workspace-server/src/authority.rs @@ -238,6 +238,13 @@ impl SqliteWorkspaceAuthority { }) } + // Contention fixtures must hold the candidate query's actual connection, + // not the API Store used only by some enrichment paths. + #[cfg(test)] + pub(crate) fn candidate_query_store_for_test(&self) -> SqliteWorkspaceStore { + self.store.clone() + } + /// Assignment self-report enrichment is a separate projection so retained /// responsibility and existing Ticket constructors stay distinct from status. pub fn ticket_assignment_tracking( @@ -3349,7 +3356,7 @@ mod tests { async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -3424,7 +3431,7 @@ mod tests { async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -3557,7 +3564,7 @@ mod tests { write_ticket(dir.path(), "00000000001J5", "Second ticket", "queued"); write_ticket(dir.path(), "00000000001J6", "Third ticket", "planning"); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), @@ -3989,7 +3996,7 @@ VALUES ('workspace-test', 'ticket', 4); async fn objective_mutations_write_sqlite_records_and_audit_events() { let dir = tempfile::tempdir().unwrap(); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), diff --git a/crates/workspace-server/src/config_source.rs b/crates/workspace-server/src/config_source.rs index 58158f464..7ac608622 100644 --- a/crates/workspace-server/src/config_source.rs +++ b/crates/workspace-server/src/config_source.rs @@ -795,7 +795,7 @@ mod tests { } async fn open_store() -> SqliteWorkspaceStore { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); store } @@ -1454,7 +1454,7 @@ mod tests { #[tokio::test] async fn invalid_candidate_is_never_persisted() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let main = current.snapshot.get(&path(MAIN_CONFIG_ENTRYPOINT)).unwrap(); @@ -1478,7 +1478,7 @@ mod tests { #[tokio::test] async fn valid_candidate_commits_snapshot_content_and_provenance_atomically() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let committed = store @@ -1496,7 +1496,7 @@ mod tests { #[tokio::test] async fn stale_cas_cannot_overwrite_newer_tree() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let request = commit_request(¤t, vec![update_main(¤t, "{ answer = 42; }")]); @@ -1514,7 +1514,7 @@ mod tests { #[tokio::test] async fn committed_content_remains_retrievable_after_later_commit() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let first = store @@ -1772,7 +1772,7 @@ mod tests { #[test] fn migration_creates_config_authority_tables() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .with_conn(|conn| { for table in [ diff --git a/crates/workspace-server/src/db_executor.rs b/crates/workspace-server/src/db_executor.rs new file mode 100644 index 000000000..1c44ef8c9 --- /dev/null +++ b/crates/workspace-server/src/db_executor.rs @@ -0,0 +1,957 @@ +//! Bounded execution of whole synchronous DB operations, independent of Tokio. +//! +//! Accepted jobs belong to the workers, not their async waiters. Cancelling a +//! waiter does not cancel or retry SQL, release capacity, or release its lease. +use std::{ + collections::BTreeSet, + future::Future, + num::NonZeroUsize, + panic::{AssertUnwindSafe, catch_unwind}, + sync::{Arc, Mutex, OnceLock, mpsc}, +}; +use tokio::sync::{Notify, oneshot}; + +use crate::{Error, Result}; + +/// The default is one FIFO worker and at most 32 waiting operations. +#[derive(Clone, Copy, Debug)] +pub struct DbExecutorLimits { + pub workers: NonZeroUsize, + pub queued: NonZeroUsize, +} + +impl Default for DbExecutorLimits { + fn default() -> Self { + Self { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(32).unwrap(), + } + } +} + +/// A request can retain its lifecycle authority until actual SQL finishes. +/// Retention happens at submission, while the caller's task-local scope exists. +pub(crate) trait DbOperationLifetime: Send + Sync { + fn retain(&self) -> Box; +} + +tokio::task_local! { + static OPERATION_LIFETIME: Arc; +} + +pub(crate) async fn with_operation_lifetime( + lifetime: Arc, + future: F, +) -> F::Output { + OPERATION_LIFETIME.scope(lifetime, future).await +} + +#[derive(Default)] +struct Accepted { + last_job_id: u64, + unfinished: BTreeSet, +} + +#[derive(Default)] +struct Tracking { + accepted: Mutex, + finished: Notify, +} + +struct Completion { + tracking: Arc, + id: u64, +} + +impl Drop for Completion { + fn drop(&mut self) { + self.tracking + .accepted + .lock() + .unwrap() + .unfinished + .remove(&self.id); + self.tracking.finished.notify_waiters(); + } +} + +struct Job { + operation: Box, + // Field drop order releases authority before completion becomes observable. + _lifetime: Option>, + lifetime_source: Option>, + _completion: Completion, + runtime: Option, + diagnostics: tracing::Dispatch, + span: tracing::Span, +} + +fn report_unknown(accepted_job_id: u64, phase: &'static str) { + tracing::error!( + code = "db_operation_outcome_unknown", + accepted_job_id, + phase, + error = %Error::DbOperationOutcomeUnknown, + "DB job panicked; outcome unknown, operation will not be retried" + ); +} + +/// Clones share the queue and actual-job budget. Workers retain only the +/// receiver, so dropping the last sender closes the queue; accepted jobs finish +/// before the dedicated threads exit. No thread is created per store open. +#[derive(Clone)] +pub struct DbExecutor { + sender: Option>, + tracking: Arc, + max_jobs: usize, +} + +impl DbExecutor { + pub fn new(limits: DbExecutorLimits) -> Result { + let max_jobs = limits + .workers + .get() + .checked_add(limits.queued.get()) + .ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?; + let (sender, receiver) = mpsc::sync_channel::(limits.queued.get()); + let receiver = Arc::new(Mutex::new(receiver)); + for index in 0..limits.workers.get() { + let receiver = receiver.clone(); + std::thread::Builder::new() + .name(format!("workspace-db-{index}")) + .spawn(move || { + loop { + let next = receiver.lock().unwrap().recv(); + let Ok(job) = next else { break }; + let Job { + operation, + _lifetime: lifetime, + lifetime_source, + _completion: completion, + runtime, + diagnostics, + span, + } = job; + // Keep diagnostics with their submitting subscriber as + // well as the runtime; no process-global override. + let _diagnostics = tracing::dispatcher::set_default(&diagnostics); + let _span = span.enter(); + // Preserve the submitting runtime for synchronous Host + // post-commit hooks, including reply/lease destructors. + // enter() supplies context only: never drive a future + // or await/nest an executor submission in a transaction. + let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter); + // A cancelled waiter's discarded result may itself have + // a destructor. Keep both the lease and actual-job slot + // until that cleanup finishes, even if it panics. + let execute = || { + if catch_unwind(AssertUnwindSafe(operation)).is_err() { + report_unknown(completion.id, "reply_cleanup"); + } + if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() { + report_unknown(completion.id, "lifetime_cleanup"); + } + }; + // Synchronous post-commit continuations can admit another + // bounded job. Transfer the original lifecycle source as + // well, so its authority survives the parent job/waiter. + // This does not inject context into detached Tokio tasks. + if let Some(source) = lifetime_source { + OPERATION_LIFETIME.sync_scope(source, execute); + } else { + execute(); + } + drop(completion); + } + }) + .map_err(|_| Error::DbExecutorUnavailable)?; + } + Ok(Self { + sender: Some(sender), + tracking: Arc::default(), + max_jobs, + }) + } + + pub fn shared() -> Self { + static SHARED: OnceLock = OnceLock::new(); + SHARED + .get_or_init(|| { + Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self { + sender: None, + tracking: Arc::default(), + max_jobs: 0, + }) + }) + .clone() + } + + /// Submit without waiting for queue capacity. The closure must contain the + /// entire synchronous operation/transaction, with no nested async runtime. + /// The submitting Tokio handle, when present, is entered for synchronous + /// post-commit hooks and worker cleanup; it is never used to block_on SQL. + pub async fn run( + &self, + operation: impl FnOnce() -> std::result::Result + Send + 'static, + ) -> std::result::Result + where + T: Send + 'static, + E: From + Send + 'static, + { + self.submit(operation)? + .await + .unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into())) + } + + /// Admit a fire-and-forget operation immediately, without spawning a Tokio + /// task or waiting for capacity. Err means the operation was not accepted; + /// Ok means the worker owns it through execution and cleanup, even after + /// the submitting task is cancelled. A panic emits a structured unknown- + /// outcome diagnostic, never an automatic retry. drain() tracks these jobs. + pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> { + self.submit(move || { + operation(); + Ok::<_, Error>(()) + }) + .map(drop) + } + + /// Shared synchronous admission: neither dropped receivers nor cancelled + /// callers remove accepted work from the queue or the actual-job budget. + fn submit( + &self, + operation: impl FnOnce() -> std::result::Result + Send + 'static, + ) -> std::result::Result>, E> + where + T: Send + 'static, + E: From + Send + 'static, + { + let sender = self + .sender + .as_ref() + .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; + let runtime = tokio::runtime::Handle::try_current().ok(); + let diagnostics = tracing::dispatcher::get_default(Clone::clone); + let span = tracing::Span::current(); + let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok(); + let lifetime = lifetime_source.as_ref().map(|source| source.retain()); + let (reply, result) = oneshot::channel(); + // Serialize admission and acceptance snapshots. Worker completion cannot + // race acceptance, and cancelled waiters never alter this actual-job cap. + { + let mut accepted = self.tracking.accepted.lock().unwrap(); + if accepted.unfinished.len() >= self.max_jobs { + return Err(Error::DbCapacityExceeded.into()); + } + let id = accepted + .last_job_id + .checked_add(1) + .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; + // Never reuse the identity of a rejected job: its destructor may run + // after another submission has acquired the admission lock. + accepted.last_job_id = id; + let job = Job { + operation: Box::new(move || { + let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| { + report_unknown(id, "operation"); + Err(Error::DbOperationOutcomeUnknown.into()) + }); + let _ = reply.send(outcome); + }), + _lifetime: lifetime, + lifetime_source, + runtime, + diagnostics, + span, + _completion: Completion { + tracking: self.tracking.clone(), + id, + }, + }; + match sender.try_send(job) { + Ok(()) => { + accepted.unfinished.insert(id); + } + Err(error) => { + // The rejected Job's completion also locks tracking; drop it only + // after releasing the admission lock. + drop(accepted); + return Err(match error { + mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded, + mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable, + } + .into()); + } + } + } + Ok(result) + } + + /// Wait for every job accepted before this barrier is first polled (not + /// later submissions). This bypasses queue capacity and survives cancelled + /// job waiters. It does not close the queue or stop new submissions. + pub async fn drain(&self) -> Result<()> { + if self.sender.is_none() { + return Err(Error::DbExecutorUnavailable); + } + let through = self.tracking.accepted.lock().unwrap().last_job_id; + loop { + let notified = self.tracking.finished.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self + .tracking + .accepted + .lock() + .unwrap() + .unfinished + .first() + .is_none_or(|id| *id > through) + { + return Ok(()); + } + notified.await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::{Connection, TransactionBehavior, ffi}; + use std::{ffi::c_void, time::Duration}; + + fn executor(queued: usize) -> DbExecutor { + DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(queued).unwrap(), + }) + .unwrap() + } + + fn database() -> (tempfile::TempDir, Connection, Connection) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite"); + let owner = Connection::open(&path).unwrap(); + owner + .execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)") + .unwrap(); + let worker = Connection::open(&path).unwrap(); + (dir, owner, worker) + } + + struct BusyGate { + waiting: Option>, + release: mpsc::Receiver<()>, + } + + // A fixture-owned callback proves SQLite actually reached its lock wait. + // Unlike busy_handler's fn pointer API, the SQLite API carries a per-call + // context, so tests need neither process-global state nor guessed sleeps. + struct BusyRegistration { + db: *mut ffi::sqlite3, + _gate: Box, + } + + impl BusyRegistration { + fn install(conn: &Connection, gate: BusyGate) -> Self { + unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 { + // The Box remains registered only while this connection and + // BusyRegistration are alive, on this same dedicated worker. + let gate = unsafe { &mut *context.cast::() }; + if let Some(waiting) = gate.waiting.take() { + let _ = waiting.send(()); + if gate.release.recv().is_ok() { + return 1; + } + } + 0 + } + let mut gate = Box::new(gate); + // SAFETY: conn owns the handle; registration is dropped before conn. + let db = unsafe { conn.handle() }; + let result = unsafe { + ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast()) + }; + assert_eq!(result, ffi::SQLITE_OK); + Self { db, _gate: gate } + } + } + + impl Drop for BusyRegistration { + fn drop(&mut self) { + // SAFETY: unregister before releasing the callback's boxed context. + unsafe { + ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut()); + } + } + } + + async fn observed(future: impl Future) -> T { + tokio::time::timeout(Duration::from_secs(5), future) + .await + .expect("DB job did not reach the expected observable state") + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() { + let (_dir, mut owner, mut worker) = database(); + let lock = owner + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let executor = executor(1); + let (waiting, blocked) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut write = Box::pin(executor.run(move || -> Result<()> { + let _busy = BusyRegistration::install( + &worker, + BusyGate { + waiting: Some(waiting), + release: permit, + }, + ); + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + Ok(()) + })); + assert!(futures::poll!(&mut write).is_pending()); + observed(blocked).await.unwrap(); + let control = tokio::spawn(async { 42 }); + assert_eq!(observed(control).await.unwrap(), 42); + // The timer itself is the contract under test, not synchronization. + observed(tokio::time::sleep(Duration::from_millis(1))).await; + assert!(futures::poll!(&mut write).is_pending()); + lock.rollback().unwrap(); + release.send(()).unwrap(); + observed(write).await.unwrap(); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + } + + struct Lease(Arc); + struct Retained(Arc); + impl DbOperationLifetime for Lease { + fn retain(&self) -> Box { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Box::new(Retained(self.0.clone())) + } + } + impl Drop for Retained { + fn drop(&mut self) { + self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() { + let (_dir, mut owner, mut worker) = database(); + let lock = owner + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let executor = executor(1); + let (waiting, blocked) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut running = Box::pin(executor.run(move || -> Result<()> { + let _busy = BusyRegistration::install( + &worker, + BusyGate { + waiting: Some(waiting), + release: permit, + }, + ); + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + Ok(()) + })); + assert!(futures::poll!(&mut running).is_pending()); + observed(blocked).await.unwrap(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap(); + let mut queued = Box::pin(with_operation_lifetime( + lease, + executor.run(move || -> Result<()> { + let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + // FIFO is observable in durable SQL, not inferred from waiter order. + let first: i64 = + tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| { + row.get(0) + })?; + assert_eq!(first, 1); + tx.execute("INSERT INTO writes VALUES (2)", [])?; + tx.commit()?; + Ok(()) + }), + )); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!(matches!( + executor.run(|| Ok::<_, Error>(())).await, + Err(Error::DbCapacityExceeded) + )); + let mut drained = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + lock.rollback().unwrap(); + release.send(()).unwrap(); + observed(running).await.unwrap(); + observed(drained).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() { + let (dir, owner, mut parent_conn) = database(); + let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap(); + let parent = executor(1); + let child = executor(1); + let child_owner = child.clone(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let source: Arc = Arc::new(Lease(count.clone())); + let (entered, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let (child_entered, child_running) = oneshot::channel(); + let (finish_child, child_permit) = mpsc::channel(); + let mut operation = Box::pin(with_operation_lifetime( + source, + parent.run(move || -> Result<()> { + let _ = entered.send(()); + let _ = permit.recv(); + let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + child_owner.dispatch(move || { + let tx = child_conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let committed: i64 = tx + .query_row("SELECT count(*) FROM writes", [], |row| row.get(0)) + .unwrap(); + assert_eq!(committed, 1, "child must see its parent's committed SQL"); + let _ = child_entered.send(()); + let _ = child_permit.recv(); + tx.execute("INSERT INTO writes VALUES (2)", []).unwrap(); + tx.commit().unwrap(); + })?; + Ok(()) + }), + )); + assert!(futures::poll!(&mut operation).is_pending()); + observed(running).await.unwrap(); + drop(operation); + release.send(()).unwrap(); + observed(child_running).await.unwrap(); + observed(parent.drain()).await.unwrap(); + assert_eq!( + count.load(std::sync::atomic::Ordering::SeqCst), + 1, + "accepted continuation lost authority when its parent finished" + ); + let mut drained = Box::pin(child.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + finish_child.send(()).unwrap(); + observed(drained).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); + } + + #[tokio::test] + async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let error = executor + .run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.execute("INSERT INTO writes VALUES (NULL)", [])?; + tx.commit()?; + Ok(()) + }) + .await + .unwrap_err(); + assert!(matches!(error, Error::Sqlite(_))); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test] + async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let error = executor + .run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + panic!("lost outcome after commit"); + }) + .await + .unwrap_err(); + assert!(matches!(error, Error::DbOperationOutcomeUnknown)); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42); + } + + #[tokio::test] + async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let error = with_operation_lifetime( + lease, + executor.run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + panic!("lost outcome before commit"); + }), + ) + .await + .unwrap_err(); + assert!(matches!(error, Error::DbOperationOutcomeUnknown)); + executor.drain().await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + struct DiscardedReplyHook { + expected_runtime: tokio::runtime::Id, + lease_count: Arc, + dropped: Option>, + } + + impl Drop for DiscardedReplyHook { + fn drop(&mut self) { + let runtime = + tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context"); + let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); + let same_runtime = runtime.id() == self.expected_runtime; + let dropped = self.dropped.take().unwrap(); + tokio::spawn(async move { + let _ = dropped.send((same_runtime, retained)); + }); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop() + { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut blocker = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut blocker).is_pending()); + observed(running).await.unwrap(); + let expected_runtime = tokio::runtime::Handle::current().id(); + let (hook, published) = oneshot::channel(); + let (finish, finishing) = mpsc::channel(); + let (dropped, reply_dropped) = oneshot::channel(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let operation_count = count.clone(); + let mut queued = Box::pin(with_operation_lifetime( + lease, + executor.run(move || -> Result<_> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + // Model synchronous Host hooks: publish only after commit, checking + // try_current and spawning without driving any future on this worker. + let runtime = tokio::runtime::Handle::try_current() + .expect("post-commit hook lost runtime context"); + assert_eq!(runtime.id(), expected_runtime); + tokio::spawn(async move { + let _ = hook.send(()); + }); + let _ = finishing.recv(); + Ok(DiscardedReplyHook { + expected_runtime, + lease_count: operation_count, + dropped: Some(dropped), + }) + }), + )); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + release.send(()).unwrap(); + observed(blocker).await.unwrap(); + observed(published).await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + finish.send(()).unwrap(); + assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1)); + observed(drain).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() { + let executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(2).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let mut releases = Vec::new(); + // Observe each worker executing before submitting the next operation, + // so queue occupancy and overload are deterministic, not timing guesses. + for _ in 0..2 { + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut operation = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut operation).is_pending()); + observed(running).await.unwrap(); + drop(operation); + releases.push(release); + } + let (finished, mut queued_finished) = oneshot::channel(); + let mut queued = Box::pin(executor.run(move || -> Result<()> { + let _ = finished.send(()); + Ok(()) + })); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + for _ in 0..8 { + let rejected_runs = rejected_runs.clone(); + assert!(matches!( + executor + .run(move || { + rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok::<_, Error>(()) + }) + .await, + Err(Error::DbCapacityExceeded) + )); + } + assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(matches!( + queued_finished.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + for release in releases { + release.send(()).unwrap(); + } + observed(drain).await.unwrap(); + observed(queued_finished).await.unwrap(); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[derive(Default)] + struct UnknownDiagnostic { + code: Option, + phase: Option, + accepted_job_id: Option, + } + + impl tracing::field::Visit for UnknownDiagnostic { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + match field.name() { + "code" => self.code = Some(value.to_owned()), + "phase" => self.phase = Some(value.to_owned()), + _ => {} + } + } + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + if field.name() == "accepted_job_id" { + self.accepted_job_id = Some(value); + } + } + fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} + } + + #[derive(Clone, Default)] + struct DiagnosticCapture(Arc>>); + + impl tracing_subscriber::Layer for DiagnosticCapture { + fn on_event( + &self, + event: &tracing::Event<'_>, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut diagnostic = UnknownDiagnostic::default(); + event.record(&mut diagnostic); + self.0.lock().unwrap().push(diagnostic); + } + } + + struct DispatchCleanup { + expected_runtime: tokio::runtime::Id, + lease_count: Arc, + started: Option>, + release: mpsc::Receiver<()>, + } + + impl Drop for DispatchCleanup { + fn drop(&mut self) { + // No panicking assertions during unwind: report observations to the + // test, and gate cleanup so drain/lifetime behavior is observable. + let same_runtime = tokio::runtime::Handle::try_current() + .is_ok_and(|runtime| runtime.id() == self.expected_runtime); + let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); + if let Some(started) = self.started.take() { + let _ = started.send((same_runtime, retained)); + } + let _ = self.release.recv(); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry() + { + use tracing_subscriber::layer::SubscriberExt; + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut blocker = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut blocker).is_pending()); + observed(running).await.unwrap(); + drop(blocker); // Cancellation does not free its running slot. + let expected_runtime = tokio::runtime::Handle::current().id(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let operation_count = count.clone(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let operation_attempts = attempts.clone(); + let (cleanup_started, cleaning_up) = oneshot::channel(); + let (finish, finishing) = mpsc::channel(); + // Model multiple tracing contexts without setting a global subscriber. + // This also prevents tracing's single-subscriber callsite optimization + // from caching an unsubscribed parallel test's Interest::never here. + let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry()); + let captured = DiagnosticCapture::default(); + let diagnostics = + tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone())); + with_operation_lifetime(lease, async { + tracing::dispatcher::with_default(&diagnostics, || { + executor.dispatch(move || { + let _cleanup = DispatchCleanup { + expected_runtime, + lease_count: operation_count, + started: Some(cleanup_started), + release: finishing, + }; + operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let tx = worker + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + tx.execute("INSERT INTO writes VALUES (1)", []).unwrap(); + tx.commit().unwrap(); + panic!("dispatched post-commit outcome lost"); + }) + }) + }) + .await + .unwrap(); // Dispatch is admitted synchronously; no waiter survives. + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let rejected_operation = rejected_runs.clone(); + assert!(matches!( + executor.dispatch(move || { + rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }), + Err(Error::DbCapacityExceeded) + )); + release.send(()).unwrap(); + assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1)); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + finish.send(()).unwrap(); + observed(drain).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); + { + let events = captured.0.lock().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!( + events[0].code.as_deref(), + Some("db_operation_outcome_unknown") + ); + assert_eq!(events[0].phase.as_deref(), Some("operation")); + assert!(events[0].accepted_job_id.is_some()); + } + let (completed, completion) = oneshot::channel(); + executor + .dispatch(move || { + let _ = completed.send(()); + }) + .unwrap(); + observed(executor.drain()).await.unwrap(); + observed(completion).await.unwrap(); + } +} diff --git a/crates/workspace-server/src/frozen_content_state_tests.rs b/crates/workspace-server/src/frozen_content_state_tests.rs index 138b83683..954d42ccf 100644 --- a/crates/workspace-server/src/frozen_content_state_tests.rs +++ b/crates/workspace-server/src/frozen_content_state_tests.rs @@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store @@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store diff --git a/crates/workspace-server/src/frozen_repository_key_tests.rs b/crates/workspace-server/src/frozen_repository_key_tests.rs index adeef98fe..798f1575d 100644 --- a/crates/workspace-server/src/frozen_repository_key_tests.rs +++ b/crates/workspace-server/src/frozen_repository_key_tests.rs @@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }; let migrated = store @@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let archived = store @@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = @@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() { // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = diff --git a/crates/workspace-server/src/frozen_runtime_binding_tests.rs b/crates/workspace-server/src/frozen_runtime_binding_tests.rs index fcf7dcc97..9c8bef71f 100644 --- a/crates/workspace-server/src/frozen_runtime_binding_tests.rs +++ b/crates/workspace-server/src/frozen_runtime_binding_tests.rs @@ -533,6 +533,7 @@ mod existing_workspace_provisioning { // This fixture contains only the schema-87 cutover authorities, not the // unrelated full Server schema. Exercise the real service/store methods. Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }) } @@ -644,6 +645,7 @@ mod existing_workspace_provisioning { drop(service); drop(store); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())), }); let service = WorkspaceSigningIdentityService::new( @@ -816,7 +818,7 @@ mod current_authority { use super::*; fn binding_store() -> (SqliteWorkspaceStore, WorkspaceRuntimeBinding) { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-key").unwrap(); store.with_conn(|conn| { diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index c0fe34c99..0b41a9a47 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -23,7 +23,10 @@ use std::{ }; use workdir::{ Workdir, WorkdirError, WorkdirSessionHandle, - http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization}, + http::{ + OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization, + WorkdirHttpAuthorizationFuture, + }, }; use worker_runtime::RuntimeWorkspaceScope; use worker_runtime::catalog::{ @@ -3477,6 +3480,38 @@ impl WorkspaceRuntimeAuthorization { } } + /// Offload the complete live-binding check and request signing operation. + /// Claims are minted inside the bounded job, not cached before admission. + pub(crate) async fn issue_async( + &self, + method: String, + path_and_query: String, + operation: String, + worker_id: Option, + body: Vec, + ) -> Result { + let executor = self.store.db_executor(); + let authorization = self.clone(); + executor + .run(move || { + Ok::<_, Error>(authorization.issue( + &method, + &path_and_query, + &operation, + worker_id.as_deref(), + &body, + )) + }) + .await + .map_err(|error| { + diagnostic( + "workspace_runtime_authorization_unavailable", + HostDiagnosticSeverity::Error, + error.to_string(), + ) + })? + } + pub(crate) fn issue( &self, method: &str, @@ -3649,25 +3684,24 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization { impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization { fn bearer_token( &self, - method: &str, - path_and_query: &str, - body: &[u8], - ) -> Result { - if let Some(authorization) = &self.workspace_authorization { - return authorization - .issue( - method, - path_and_query, - workspace_runtime_operation(method, path_and_query), - None, - body, + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let authorization = self.clone(); + Box::pin(async move { + if let Some(authorizer) = &authorization.workspace_authorization { + let operation = workspace_runtime_operation(&method, &path_and_query).to_string(); + return authorizer + .issue_async(method, path_and_query, operation, None, body) + .await + .map_err(|error| WorkdirError::Unavailable(error.message)); + } + authorization.fallback_bearer_token.ok_or_else(|| { + WorkdirError::Unavailable( + "remote Runtime does not have bearer authorization configured".to_string(), ) - .map_err(|error| WorkdirError::Unavailable(error.message)); - } - self.fallback_bearer_token.clone().ok_or_else(|| { - WorkdirError::Unavailable( - "remote Runtime does not have bearer authorization configured".to_string(), - ) + }) }) } } @@ -6348,6 +6382,79 @@ mod tests { use std::sync::{Arc, Mutex}; use std::thread; + #[tokio::test] + async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() { + use crate::db_executor::{DbExecutor, DbExecutorLimits}; + use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding}; + use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore; + + let executor = DbExecutor::new(DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::in_memory_for_test() + .unwrap() + .with_db_executor(executor.clone()), + ); + let service = WorkspaceSigningIdentityService::new( + store.clone(), + Arc::new(InMemoryWorkspaceSigningMaterialStore::default()), + ); + let authorization = RemoteWorkdirAuthorization { + workspace_authorization: Some(WorkspaceRuntimeAuthorization::new( + store, + service, + "https://backend.test", + WorkspaceRuntimeBinding { + workspace_id: "workspace".into(), + runtime_id: "runtime".into(), + display_name: "Runtime".into(), + base_url: "https://runtime.test".into(), + public_key_fingerprint: "unused".into(), + public_key: "unused".into(), + binding_id: "binding".into(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: "1".into(), + updated_at: "1".into(), + revoked_at: None, + }, + )), + fallback_bearer_token: Some("must-not-be-used".into()), + }; + + // Hold the running slot and fill the only queue slot. The callback must + // reject admission, not run inline or silently switch authentication. + let (release, wait) = std::sync::mpsc::channel::<()>(); + let (started, running) = tokio::sync::oneshot::channel(); + executor + .dispatch(move || { + let _ = started.send(()); + let _ = wait.recv_timeout(Duration::from_secs(5)); + }) + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), running) + .await + .unwrap() + .unwrap(); + executor.dispatch(|| {}).unwrap(); + let error = authorization + .bearer_token( + "POST".into(), + "/v1/workdir-sessions/s/operations".into(), + b"{}".to_vec(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string() + )); + drop(release); + executor.drain().await.unwrap(); + } + #[test] fn ticket_worker_spawn_intent_uses_worker_wire_role() { let intent = WorkerSpawnIntent::TicketRole { diff --git a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs index f68964f35..29c0cb52c 100644 --- a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs +++ b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs @@ -32,7 +32,8 @@ struct Fixture { impl Fixture { async fn new() -> Self { let temp = tempfile::tempdir().unwrap(); - let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap()); + let store = + Arc::new(SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap()); store .upsert_account(&AccountRecord { account_id: "owner".into(), diff --git a/crates/workspace-server/src/legacy_evidence_column_tests.rs b/crates/workspace-server/src/legacy_evidence_column_tests.rs index a59f2a0ca..33994b292 100644 --- a/crates/workspace-server/src/legacy_evidence_column_tests.rs +++ b/crates/workspace-server/src/legacy_evidence_column_tests.rs @@ -45,7 +45,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() { let artifacts = evidence_rows(&conn, "artifacts"); let receipts = evidence_rows(&conn, "repository_secret_legacy_receipts"); drop(conn); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); upgraded .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -73,7 +73,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() { Ok(()) }) .unwrap(); - for store in [upgraded, SqliteWorkspaceStore::in_memory().unwrap()] { + for store in [upgraded, SqliteWorkspaceStore::in_memory_for_test().unwrap()] { store.with_conn(|conn| { let mut stmt = conn.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")?; let objects = stmt.query_map([],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?)))?.collect::>>()?; diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index fefe24d92..eb113b079 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -10,6 +10,7 @@ pub mod backend_job; pub mod companion; pub mod config; pub mod config_source; +pub mod db_executor; pub mod feature_storage; pub mod hosts; pub mod memory_backend; @@ -159,6 +160,12 @@ pub enum Error { WorkspaceSigningIdentity { code: String, message: String }, #[error("Workspace Drive error: {0}")] Drive(#[from] workspace_drive::Error), + #[error("DB execution capacity exceeded")] + DbCapacityExceeded, + #[error("DB executor unavailable")] + DbExecutorUnavailable, + #[error("DB operation outcome unknown; do not automatically retry")] + DbOperationOutcomeUnknown, #[error("store error: {0}")] Store(String), } diff --git a/crates/workspace-server/src/memory_staging.rs b/crates/workspace-server/src/memory_staging.rs index ff8caadf7..631d1b33e 100644 --- a/crates/workspace-server/src/memory_staging.rs +++ b/crates/workspace-server/src/memory_staging.rs @@ -217,7 +217,7 @@ mod tests { async fn authority() -> (TempDir, SqliteWorkspaceAuthority) { let temp = TempDir::new().unwrap(); let db_path = temp.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), diff --git a/crates/workspace-server/src/migration_entrypoint_tests.rs b/crates/workspace-server/src/migration_entrypoint_tests.rs index d0eb2375d..41329a730 100644 --- a/crates/workspace-server/src/migration_entrypoint_tests.rs +++ b/crates/workspace-server/src/migration_entrypoint_tests.rs @@ -32,7 +32,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() { drop(conn); for attempt in 0..3 { let error = match attempt { - 0 => SqliteWorkspaceStore::open(&path).err().unwrap(), + 0 => SqliteWorkspaceStore::open_for_test(&path).err().unwrap(), 1 => SqliteWorkspaceStore::migration_plan(&path).unwrap_err(), _ => SqliteWorkspaceStore::migrate_database(&path).unwrap_err(), }; @@ -62,7 +62,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() { ) .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); assert_eq!(conn.query_row("SELECT input_fingerprint FROM worker_removal_operations WHERE operation_id='removal'",[],|r|r.get::<_,String>(0))?,"legacy-receipt"); @@ -109,7 +109,7 @@ fn migration_dry_run_and_apply_share_the_original_repository_secret_authority() assert!(!table_exists(&conn, "server_secret_objects").unwrap()); drop(conn); assert_eq!(SqliteWorkspaceStore::migrate_database(&path).unwrap(), plan); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?,LATEST_SCHEMA_VERSION); let (nonce, mut sealed): (Vec,Vec) = conn.query_row( diff --git a/crates/workspace-server/src/repository_access.rs b/crates/workspace-server/src/repository_access.rs index 304909848..c8a13c3f4 100644 --- a/crates/workspace-server/src/repository_access.rs +++ b/crates/workspace-server/src/repository_access.rs @@ -2290,7 +2290,7 @@ mod tests { ) { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); let timestamp = now(); for workspace_id in ["workspace-a", "workspace-b"] { futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord { diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index aa6ca692d..4aa310885 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -1116,7 +1116,7 @@ mod tests { WorkerId::from_legacy_u64(1) } fn setup() -> SqliteWorkspaceStore { - let s = SqliteWorkspaceStore::in_memory().unwrap(); + let s = SqliteWorkspaceStore::in_memory_for_test().unwrap(); s.with_conn(|c| { c.execute( "INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) \ diff --git a/crates/workspace-server/src/runtime_subscription.rs b/crates/workspace-server/src/runtime_subscription.rs index 3a7fdd0bd..998786c38 100644 --- a/crates/workspace-server/src/runtime_subscription.rs +++ b/crates/workspace-server/src/runtime_subscription.rs @@ -965,7 +965,7 @@ async fn connect_runtime( let mut request = endpoint .into_client_request() .map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?; - if let Some(token) = runtime_token(config, workspace_id)? { + if let Some(token) = runtime_token(config, workspace_id).await? { request.headers_mut().insert( "authorization", format!("Bearer {token}") @@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String { format!("{base}/v1/protocol/ws") } } -fn runtime_token( +async fn runtime_token( config: &RemoteRuntimeConfig, _workspace_id: &str, ) -> Result, String> { if let Some(authorization) = config.workspace_authorization.as_ref() { return authorization - .issue("GET", "/v1/protocol/ws", "workers:list", None, &[]) + .issue_async( + "GET".to_string(), + "/v1/protocol/ws".to_string(), + "workers:list".to_string(), + None, + Vec::new(), + ) + .await .map(Some) .map_err(|error| error.message); } diff --git a/crates/workspace-server/src/schema_equivalence_tests.rs b/crates/workspace-server/src/schema_equivalence_tests.rs index 37f5e218a..4063a675b 100644 --- a/crates/workspace-server/src/schema_equivalence_tests.rs +++ b/crates/workspace-server/src/schema_equivalence_tests.rs @@ -38,8 +38,8 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("upgraded.db"); prepare_retained_schema(&path, 85); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let mut mismatches = Vec::new(); for table in [ "workspace_tags", diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 5ed22bcc8..a37c91975 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -885,7 +885,7 @@ impl ExternalProviderWorkdirSession { } } - fn audit_operation( + async fn audit_operation( &self, operation_id: &ExternalWorkdirOperationId, operation_kind: &str, @@ -894,25 +894,34 @@ impl ExternalProviderWorkdirSession { let Some(audit) = &self.audit else { return Ok(()); }; + let audit = audit.clone(); + let operation_id = operation_id.clone(); + let operation_kind = operation_kind.to_owned(); + let outcome = outcome.to_owned(); + let connection = self.connection.clone(); audit .store - .record_external_workdir_audit( - &audit.workspace_id, - "worker", - &audit.actor_key, - "external_workdir_operation", - &self.connection.grant_id, - Some(operation_id.as_str()), - outcome, - &format!( - "workdir_id={} attachment_alias={} operation_kind={} generation={}", - self.connection.workdir_id, - audit.attachment_alias, - operation_kind, - self.connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) + .db_executor() + .run(move || { + audit.store.record_external_workdir_audit( + &audit.workspace_id, + "worker", + &audit.actor_key, + "external_workdir_operation", + &connection.grant_id, + Some(operation_id.as_str()), + &outcome, + &format!( + "workdir_id={} attachment_alias={} operation_kind={} generation={}", + connection.workdir_id, + audit.attachment_alias, + operation_kind, + connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await .map_err(|error| { workdir::WorkdirError::Unavailable(format!( "External Workdir operation audit failed: {error}" @@ -960,7 +969,8 @@ impl ExternalProviderWorkdirSession { } let operation_id = ExternalWorkdirOperationId::new(format!("op-{}", Uuid::now_v7())) .map_err(workdir::WorkdirError::InvalidArgument)?; - self.audit_operation(&operation_id, operation_kind, "requested")?; + self.audit_operation(&operation_id, operation_kind, "requested") + .await?; let (response, receive) = tokio::sync::oneshot::channel(); let mut cancel_guard = ExternalOperationCancelGuard { sender: self.connection.sender.clone(), @@ -977,7 +987,8 @@ impl ExternalProviderWorkdirSession { .await .is_err() { - self.audit_operation(&operation_id, operation_kind, "provider_unavailable")?; + self.audit_operation(&operation_id, operation_kind, "provider_unavailable") + .await?; return Err(workdir::WorkdirError::Unavailable( "External Workdir provider disconnected".to_string(), )); @@ -1007,7 +1018,8 @@ impl ExternalProviderWorkdirSession { } else { "failed" }, - )?; + ) + .await?; result } @@ -1577,6 +1589,7 @@ fn orchestrator_attention_state( #[derive(Clone)] pub struct WorkspaceApi { + host_executors: HostExecutors, pub(crate) config: ServerConfig, pub(crate) store: Arc, config_store: Arc, @@ -1675,6 +1688,7 @@ impl ServerApiContractService { struct WorkerRemovalService { workspace_id: String, store: Arc, + host_executors: HostExecutors, runtime: Weak, workdir_sessions: Arc>, workdir_session_locks: Arc>>>>, @@ -1688,6 +1702,7 @@ impl WorkerRemovalService { Self { workspace_id: api.config.workspace_id.clone(), store: api.store.clone(), + host_executors: api.host_executors.clone(), runtime: Arc::downgrade(&api.runtime), workdir_sessions: api.workdir_sessions.clone(), workdir_session_locks: api.workdir_session_locks.clone(), @@ -1853,6 +1868,30 @@ impl WorkerRemovalService { target_runtime_id: &str, target_worker_id: &str, reason: &str, + ) -> std::result::Result { + let service = self.clone(); + let runtime_id = target_runtime_id.to_owned(); + let worker_id = target_worker_id.to_owned(); + let reason = reason.to_owned(); + run_host_workflow( + self.host_executors.host().map_err(|e| e.to_string())?, + async move { + service + .execute_async_on_host(source, &runtime_id, &worker_id, &reason) + .await + .map_err(Error::Store) + }, + ) + .await + .map_err(|error| error.to_string()) + } + + async fn execute_async_on_host( + &self, + source: crate::worker_source::VerifiedWorkerMutationSource, + target_runtime_id: &str, + target_worker_id: &str, + reason: &str, ) -> std::result::Result { let reason = reason.trim(); if reason.is_empty() || reason.len() > 512 { @@ -1924,11 +1963,39 @@ impl WorkerRemovalService { )); } - self.execute_target_removal(&runtime, &target, reason, None) + self.execute_target_removal_on_host(&runtime, &target, reason, None) .await } async fn execute_target_removal( + &self, + _runtime: &RuntimeRegistry, + target: &RuntimeWorkerRef, + reason: &str, + parent_workspace_deletion_operation_id: Option<&str>, + ) -> std::result::Result { + let runtime = self + .runtime + .upgrade() + .ok_or_else(|| "Workspace Runtime registry is unavailable".to_string())?; + let service = self.clone(); + let target = target.clone(); + let reason = reason.to_owned(); + let parent = parent_workspace_deletion_operation_id.map(str::to_owned); + run_host_workflow( + self.host_executors.host().map_err(|e| e.to_string())?, + async move { + service + .execute_target_removal_on_host(&runtime, &target, &reason, parent.as_deref()) + .await + .map_err(Error::Store) + }, + ) + .await + .map_err(|error| error.to_string()) + } + + async fn execute_target_removal_on_host( &self, runtime: &RuntimeRegistry, target: &RuntimeWorkerRef, @@ -2177,6 +2244,7 @@ use workspace_admission::WorkspaceAdmission; #[derive(Clone)] pub struct WorkspaceServerApi { + host_executors: HostExecutors, template: Arc, store: Arc, catalog: WorkspaceCatalogService, @@ -2207,6 +2275,7 @@ impl WorkspaceServerApi { workspace_signing_material_root(&template.database_path), )); Self { + host_executors: HostExecutors::shared(), template: Arc::new(template), catalog: WorkspaceCatalogService::new(store.clone(), signing_materials.clone()), signing_materials, @@ -2222,6 +2291,23 @@ impl WorkspaceServerApi { } } + #[cfg(test)] + fn new_for_test(template: ServerConfig, store: Arc) -> Self { + let mut api = Self::new(template, store); + api.host_executors = HostExecutors::default(); + api + } + + // Own the complete synchronous store stage; callers retain their async + // lifecycle guards across it and never split a transaction around await. + async fn execute_store_stage( + &self, + operation: impl FnOnce(Arc) -> Result + Send + 'static, + ) -> Result { + let store = self.store.clone(); + self.store.db_executor().run(move || operation(store)).await + } + async fn admission_for_workspace(&self, workspace_id: &str) -> Arc { workspace_admission_gate(&self.admission_gates, workspace_id).await } @@ -2234,11 +2320,22 @@ impl WorkspaceServerApi { let Some(workspace) = self.store.get_workspace(workspace_id).await? else { return Ok(None); }; - let repositories = self.store.list_repositories(workspace_id)?; + let store = self.store.clone(); + let id = workspace_id.to_owned(); + let repositories = self + .store + .db_executor() + .run(move || store.list_repositories(&id)) + .await?; let config = self .template .for_catalog_workspace(&workspace, repositories)?; - let api = WorkspaceApi::new(config, self.store.clone()).await?; + let api = WorkspaceApi::new_with_host_executors( + config, + self.store.clone(), + self.host_executors.clone(), + ) + .await?; apis.insert(workspace_id.to_string(), api.clone()); Ok(Some(api)) } @@ -2251,34 +2348,46 @@ impl WorkspaceServerApi { drop(running); let api = self.clone(); tokio::spawn(async move { - if api.execute_workspace_deletion(&operation_id).await.is_err() { - let operation = api - .store - .workspace_deletion_operation_for_recovery(&operation_id) - .ok() - .flatten(); - let child_operation_ids = operation - .as_ref() - .map(|operation| operation.child_operation_ids.as_slice()) - .unwrap_or_default(); - let blockers = operation - .as_ref() - .map(|operation| operation.blockers.as_slice()) - .unwrap_or_default(); - let _ = api.store.update_workspace_deletion_operation( - &operation_id, - WorkspaceDeletionState::Failed, - child_operation_ids, - blockers, - Some("workspace_deletion_execution_failed"), - ); + if let Err(error) = api.execute_workspace_deletion(&operation_id).await { + tracing::warn!(%error, %operation_id, "Workspace deletion execution did not complete"); + if !matches!(error, Error::DbOperationOutcomeUnknown) { + let id = operation_id.clone(); + let _ = api + .execute_store_stage(move |store| { + let operation = store + .workspace_deletion_operation_for_recovery(&id) + .ok() + .flatten(); + let child_operation_ids = operation + .as_ref() + .map(|operation| operation.child_operation_ids.as_slice()) + .unwrap_or_default(); + let blockers = operation + .as_ref() + .map(|operation| operation.blockers.as_slice()) + .unwrap_or_default(); + store + .update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Failed, + child_operation_ids, + blockers, + Some("workspace_deletion_execution_failed"), + ) + .map(drop) + }) + .await; + } } api.running_deletions.lock().await.remove(&operation_id); }); } async fn recover_workspace_deletions(&self) -> Result<()> { - for operation_id in self.store.resumable_workspace_deletion_operation_ids()? { + let operations = self + .execute_store_stage(|store| store.resumable_workspace_deletion_operation_ids()) + .await?; + for operation_id in operations { self.schedule_workspace_deletion(operation_id).await; } Ok(()) @@ -2289,16 +2398,20 @@ impl WorkspaceServerApi { actor_account_id: &str, workspace_id: &str, ) -> Result { + let actor = actor_account_id.to_owned(); + let id = workspace_id.to_owned(); let mut preflight = self - .store - .workspace_deletion_preflight(actor_account_id, workspace_id)?; + .execute_store_stage(move |store| store.workspace_deletion_preflight(&actor, &id)) + .await?; let Some(api) = self.api_for_workspace(workspace_id).await? else { return Err(Error::InvalidInput("Workspace does not exist".to_string())); }; - for registry_worker in self - .store - .list_worker_registry(workspace_id, i64::MAX as usize)? - { + let id = workspace_id.to_owned(); + let workers = self + .execute_store_stage(move |store| store.list_worker_registry(&id, i64::MAX as usize)) + .await?; + api.host_executors.host()?.run(move || -> Result<_> { + for registry_worker in workers { let worker_key = registry_worker.display_name; match api.runtime.worker(®istry_worker.worker) { Ok(worker) if worker.state == "stopped" => {} @@ -2321,23 +2434,30 @@ impl WorkspaceServerApi { crate::workspace_deletion::bound_workspace_deletion_blockers(&mut preflight.blockers); preflight.can_delete = preflight.blockers.is_empty(); Ok(preflight) + }).await } async fn execute_workspace_deletion( &self, operation_id: &str, ) -> Result { - let prior = self - .store - .workspace_deletion_operation_for_recovery(operation_id)? - .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; - let operation = self.store.update_workspace_deletion_operation( - operation_id, - WorkspaceDeletionState::Running, - &prior.child_operation_ids, - &[], - None, - )?; + let id = operation_id.to_owned(); + let operation = self + .execute_store_stage(move |store| { + let prior = store + .workspace_deletion_operation_for_recovery(&id)? + .ok_or_else(|| { + Error::InvalidInput("Workspace deletion operation".to_string()) + })?; + store.update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Running, + &prior.child_operation_ids, + &[], + None, + ) + }) + .await?; let api = self .api_for_workspace(&operation.workspace_id) .await? @@ -2346,10 +2466,13 @@ impl WorkspaceServerApi { api.fence_drive().await?; let mut child_operation_ids = operation.child_operation_ids.clone(); let mut blockers = Vec::new(); - for worker in self - .store - .list_worker_registry(&operation.workspace_id, i64::MAX as usize)? - { + let workspace_id = operation.workspace_id.clone(); + let workers = self + .execute_store_stage(move |store| { + store.list_worker_registry(&workspace_id, i64::MAX as usize) + }) + .await?; + for worker in workers { let worker_key = worker.display_name.clone(); let target = worker.worker; let response = WorkerRemovalService::new(&api) @@ -2361,13 +2484,17 @@ impl WorkspaceServerApi { ) .await .map_err(Error::Store)?; + let id = operation_id.to_owned(); child_operation_ids = self - .store - .workspace_deletion_operation_for_recovery(operation_id)? - .ok_or_else(|| { - Error::Store("Workspace deletion operation disappeared".to_string()) - })? - .child_operation_ids; + .execute_store_stage(move |store| { + store + .workspace_deletion_operation_for_recovery(&id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + }) + .map(|operation| operation.child_operation_ids) + }) + .await?; if response.status != 200 { blockers.push(WorkspaceDeletionBlocker { kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, @@ -2379,22 +2506,36 @@ impl WorkspaceServerApi { } if blockers.is_empty() { - for workdir in self - .store - .list_workdir_registry(&operation.workspace_id, i64::MAX as usize)? - { - match execute_workdir_removal_for_workspace_deletion( - &api, - &workdir.workdir_id, - operation_id, - ) { + let workspace_id = operation.workspace_id.clone(); + let workdirs = self + .execute_store_stage(move |store| { + store.list_workdir_registry(&workspace_id, i64::MAX as usize) + }) + .await?; + for workdir in workdirs { + let removal_api = api.clone(); + let workdir_id = workdir.workdir_id.clone(); + let parent_id = operation_id.to_owned(); + match api + .host_executors + .host()? + .run(move || { + execute_workdir_removal_for_workspace_deletion( + &removal_api, + &workdir_id, + &parent_id, + ) + }) + .await + { Ok(child) => { + let id = operation_id.to_owned(); + let child_id = child.operation_id.clone(); child_operation_ids = self - .store - .append_workspace_deletion_child_operation( - operation_id, - &child.operation_id, - )? + .execute_store_stage(move |store| { + store.append_workspace_deletion_child_operation(&id, &child_id) + }) + .await? .child_operation_ids; if child.state != WorkdirRemovalOperationState::Completed || child.disposition != Some(WorkdirRemovalDisposition::Removed) @@ -2430,29 +2571,40 @@ impl WorkspaceServerApi { } if !blockers.is_empty() { - return self.store.update_workspace_deletion_operation( - operation_id, - WorkspaceDeletionState::Blocked, - &child_operation_ids, - &blockers, - None, - ); + let id = operation_id.to_owned(); + return self + .execute_store_stage(move |store| { + store.update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Blocked, + &child_operation_ids, + &blockers, + None, + ) + }) + .await; } api.purge_drive().await?; let feature_storage = api.feature_storage.clone(); - tokio::task::spawn_blocking(move || feature_storage.delete()) - .await - .map_err(|error| { - Error::Store(format!("Feature storage deletion task failed: {error}")) - })? - .map_err(|error| { - Error::Store(format!( - "failed to delete Workspace Feature storage: {error}" - )) - })?; - WorkspaceSigningIdentityService::new(self.store.clone(), self.signing_materials.clone()) - .delete_material(&operation.workspace_id)?; - let completed = self.store.finalize_workspace_deletion(operation_id)?; + api.store + .db_executor() + .run(move || { + feature_storage.delete().map_err(|error| { + Error::Store(format!( + "failed to delete Workspace Feature storage: {error}" + )) + }) + }) + .await?; + let materials = self.signing_materials.clone(); + let id = operation_id.to_owned(); + let completed = self + .execute_store_stage(move |store| { + WorkspaceSigningIdentityService::new(store.clone(), materials) + .delete_material(&operation.workspace_id)?; + store.finalize_workspace_deletion(&id) + }) + .await?; self.routers.lock().await.remove(&completed.workspace_id); if let Some(handle) = self .hook_handles @@ -2499,15 +2651,28 @@ impl WorkspaceServerApi { async fn shutdown_feature_storage(&self) -> Result<()> { let apis = self.apis.lock().await.values().cloned().collect::>(); for api in apis { - api.feature_storage.shutdown().map_err(|error| { - Error::Store(format!("failed to shut down Feature storage: {error}")) - })?; + api.worker_projection.shutdown_async().await; + let storage = api.feature_storage.clone(); + api.store + .db_executor() + .run(move || { + storage.shutdown().map_err(|error| { + Error::Store(format!("failed to shut down Feature storage: {error}")) + }) + }) + .await?; } Ok(()) } async fn preload(&self) -> Result<()> { - for workspace in self.store.list_workspaces()? { + let store = self.store.clone(); + let workspaces = self + .store + .db_executor() + .run(move || store.list_workspaces()) + .await?; + for workspace in workspaces { let _ = self .router_for_workspace(&workspace.workspace_id) .await? @@ -2577,7 +2742,7 @@ async fn resolve_server_actor( headers: &HeaderMap, ) -> std::result::Result, Error> { let cookie_name = auth_public_config(api.template.as_ref()).cookie_name; - resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await + resolve_request_actor(api.store.clone(), headers, &cookie_name).await } fn signed_request_target(uri: &Uri) -> &str { @@ -2616,6 +2781,11 @@ fn runtime_request_proof_rejection( workspace_id: &str, error: &crate::worker_source::RuntimeRequestProofError, ) -> Response { + let status = if error.is_availability_error() { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::UNAUTHORIZED + }; // Never format an error or raw claims. IDs are fixed-length fingerprints; // verification labels describe proof-check progress, not authorization. let path = path.split('?').next().unwrap_or(path); @@ -2627,7 +2797,7 @@ fn runtime_request_proof_rejection( target: "yoi::auth", event = "runtime_request_proof_rejected", method, path = log_path, workspace_id, request_id, - status = 401, + status = status.as_u16(), reason = error.reason, proof_verification = error.verification, claimed_runtime_id_hash = error.runtime_id_hash.as_deref(), @@ -2642,8 +2812,12 @@ fn runtime_request_proof_rejection( ); repository_api_rejection( path, - StatusCode::UNAUTHORIZED, - "invalid runtime request proof", + status, + if error.is_availability_error() { + "Runtime authentication authority unavailable" + } else { + "invalid runtime request proof" + }, ) } @@ -2718,7 +2892,7 @@ async fn authorize_scoped_workspace_request( worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION }; let source = crate::worker_source::verify_runtime_request_source_proof_with_clock( - api.store.as_ref(), + api.store.clone(), api.template.as_ref(), &proof, workspace_id, @@ -2883,41 +3057,38 @@ async fn authorize_workspace_api_request( if !matches!( *request.method(), Method::GET | Method::HEAD | Method::OPTIONS - ) && !api - .store - .get_workspace(&workspace_id) - .await - .ok() - .flatten() - .is_some_and(|workspace| workspace.state == "active") - { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); + ) { + let workspace = match api.store.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(error) => return server_error_response(error), + }; + if !workspace.is_some_and(|workspace| workspace.state == "active") { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); + } } return next.run(request).await; } let AuthConfig::Passkey { cookie_name, .. } = &api.config.auth; - let actor = match crate::auth::resolve_request_actor( - api.store.as_ref(), - request.headers(), - cookie_name, - ) - .await - { - Ok(Some(actor)) => actor, - Ok(None) if public_server_api => return next.run(request).await, - Ok(None) | Err(_) => { - return repository_api_rejection( - &request_path, - StatusCode::UNAUTHORIZED, - "authentication required", - ); - } - }; + let actor = + match crate::auth::resolve_request_actor(api.store.clone(), request.headers(), cookie_name) + .await + { + Ok(Some(actor)) => actor, + Ok(None) if public_server_api => return next.run(request).await, + Err(error) => return server_error_response(error), + Ok(None) => { + return repository_api_rejection( + &request_path, + StatusCode::UNAUTHORIZED, + "authentication required", + ); + } + }; let cookie_authenticated = matches!(actor.auth_method, ActorAuthMethod::BrowserSession); let mutating = !matches!( *request.method(), @@ -2940,20 +3111,18 @@ async fn authorize_workspace_api_request( } } request.extensions_mut().insert(actor); - if mutating - && !api - .store - .get_workspace(&workspace_id) - .await - .ok() - .flatten() - .is_some_and(|workspace| workspace.state == "active") - { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); + if mutating { + let workspace = match api.store.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(error) => return server_error_response(error), + }; + if !workspace.is_some_and(|workspace| workspace.state == "active") { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); + } } next.run(request).await } @@ -3004,7 +3173,7 @@ fn workspace_request_is_callback(method: &Method, path: &str, workspace_id: &str async fn dispatch_workspace_request( State(api): State, - mut request: Request, + request: Request, ) -> Response { let path = request.uri().path().to_owned(); let workspace_id = scoped_workspace_id(&path); @@ -3030,6 +3199,23 @@ async fn dispatch_workspace_request( } else { None }; + if let Some(lease) = _request_lease { + crate::db_executor::with_operation_lifetime( + Arc::new(lease), + dispatch_admitted_workspace_request(api, request), + ) + .await + } else { + dispatch_admitted_workspace_request(api, request).await + } +} + +async fn dispatch_admitted_workspace_request( + api: WorkspaceServerApi, + mut request: Request, +) -> Response { + let path = request.uri().path().to_owned(); + let workspace_id = scoped_workspace_id(&path); if let Some(workspace_id) = workspace_id && (path.starts_with("/api/w/") || path.starts_with("/api/runtime/v1/workspaces/") @@ -3046,7 +3232,13 @@ async fn dispatch_workspace_request( Err(error) => return server_error_response(error), } } else { - let workspaces = match api.store.list_workspaces() { + let store = api.store.clone(); + let workspaces = match api + .store + .db_executor() + .run(move || store.list_workspaces()) + .await + { Ok(workspaces) => workspaces, Err(error) => return server_error_response(error), }; @@ -3322,6 +3514,19 @@ impl WorkspaceApi { } pub async fn new(config: ServerConfig, store: Arc) -> Result { + Self::new_with_host_executors(config, store, HostExecutors::shared()).await + } + + #[cfg(test)] + async fn new_for_test(config: ServerConfig, store: Arc) -> Result { + Self::new_with_host_executors(config, store, HostExecutors::default()).await + } + + async fn new_with_host_executors( + config: ServerConfig, + store: Arc, + host_executors: HostExecutors, + ) -> Result { let resource_broker = BackendResourceBroker::default(); if store.get_workspace(&config.workspace_id).await?.is_none() { return Err(crate::Error::Config(format!( @@ -3398,6 +3603,7 @@ impl WorkspaceApi { store, Arc::new(execution_backend), resource_broker, + host_executors, Some(worker_remove_dispatcher), ) .await?; @@ -3458,6 +3664,7 @@ impl WorkspaceApi { store, execution_backend, BackendResourceBroker::default(), + HostExecutors::default(), None, ) .await @@ -3468,6 +3675,7 @@ impl WorkspaceApi { store: Arc, execution_backend: Arc, resource_broker: BackendResourceBroker, + host_executors: HostExecutors, worker_remove_dispatcher: Option< Arc, >, @@ -3478,8 +3686,18 @@ impl WorkspaceApi { config.workspace_id ))); } - import_configured_repositories(store.as_ref(), &config)?; - config.repositories = load_configured_repositories_from_store(store.as_ref(), &config)?; + let repository_store = store.clone(); + let repository_config = config.clone(); + config.repositories = store + .db_executor() + .run(move || { + import_configured_repositories(repository_store.as_ref(), &repository_config)?; + load_configured_repositories_from_store( + repository_store.as_ref(), + &repository_config, + ) + }) + .await?; let embedded_runtime = EmbeddedWorkerRuntime::new_fs_store_with_execution_backend( config.workspace_id.clone(), config.embedded_runtime_store_root.clone(), @@ -3494,20 +3712,34 @@ impl WorkspaceApi { RuntimeSubscriptionBroker::new(config.workspace_id.clone()); runtime_subscription_broker .register_embedded_runtime(embedded_runtime_id, embedded_subscription_runtime); - let config_store = Arc::new(crate::SqliteWorkspaceStore::open( - config.database_path.clone(), - )?); - let repository_secrets = Arc::new(RepositorySecretService::open( - config_store.clone(), - &config.database_path, - )?); + let database_path = config.database_path.clone(); + let executor = store.db_executor(); + let config_executor = executor.clone(); + let (config_store, repository_secrets) = executor + .run(move || { + let config_store = Arc::new( + crate::SqliteWorkspaceStore::open(database_path.clone())? + .with_db_executor(config_executor), + ); + let repository_secrets = Arc::new(RepositorySecretService::open( + config_store.clone(), + &database_path, + )?); + Ok::<_, Error>((config_store, repository_secrets)) + }) + .await?; let signing_materials: Arc = Arc::new(FsWorkspaceSigningMaterialStore::new( workspace_signing_material_root(&config.database_path), )); let signing_identities = WorkspaceSigningIdentityService::new(store.clone(), signing_materials); - signing_identities.get_validated(&config.workspace_id)?; + let identities = signing_identities.clone(); + let workspace_id = config.workspace_id.clone(); + store + .db_executor() + .run(move || identities.get_validated(&workspace_id)) + .await?; let backend_url = config .backend_base_url .clone() @@ -3550,11 +3782,19 @@ impl WorkspaceApi { .with_provider(Arc::new(RuntimeConfigSchemaProvider)) .with_provider(Arc::new(RepositoryAccessConfigSchemaProvider)) .with_provider(Arc::new(skills::SkillConfigSchemaProvider)); - config_store.ensure_workspace_config_materialized_with_schema( - &config.workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), - config_schema_registry.compose()?, - )?; + let materialization_store = config_store.clone(); + let workspace_id = config.workspace_id.clone(); + let schema = config_schema_registry.compose()?; + store + .db_executor() + .run(move || { + materialization_store.ensure_workspace_config_materialized_with_schema( + &workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + schema, + ) + }) + .await?; let worker_projection = crate::worker_projection::WorkerProjectionService::new( config.workspace_id.clone(), store.clone(), @@ -3565,18 +3805,28 @@ impl WorkspaceApi { .await .map_err(|error| Error::InvalidInput(error.to_string()))?; } - let feature_storage = crate::FeatureStorage::for_server_database(&config.database_path) - .and_then(|storage| storage.workspace(&config.workspace_id)) - .map_err(|error| { - Error::Store(format!("failed to initialize Feature storage: {error}")) - })?; - let subjektiv_registration = crate::subjektiv::SubjektivStore::register(&feature_storage) - .map_err(|error| { - Error::Store(format!("failed to register subjektiv storage: {error}")) - })?; + let database_path = config.database_path.clone(); + let workspace_id = config.workspace_id.clone(); + let (feature_storage, subjektiv_registration, authority) = store + .db_executor() + .run(move || { + let feature_storage = crate::FeatureStorage::for_server_database(&database_path) + .and_then(|storage| storage.workspace(&workspace_id)) + .map_err(|error| { + Error::Store(format!("failed to initialize Feature storage: {error}")) + })?; + let registration = crate::subjektiv::SubjektivStore::register(&feature_storage) + .map_err(|error| { + Error::Store(format!("failed to register subjektiv storage: {error}")) + })?; + let authority = SqliteWorkspaceAuthority::new(database_path, workspace_id)?; + Ok::<_, Error>((feature_storage, registration, authority)) + }) + .await?; let drive_host = Arc::new(crate::workspace_drive_host::WorkspaceDriveHost::new(&feature_storage).await?); let mut api = Self { + host_executors, drive_host, config_store, repository_secrets, @@ -3586,10 +3836,7 @@ impl WorkspaceApi { crate::prompt_settings::WorkspacePromptProjectionCache::default(), feature_storage, subjektiv_registration, - authority: SqliteWorkspaceAuthority::new( - config.database_path.clone(), - config.workspace_id.clone(), - )?, + authority, _worker_projection_shutdown: Arc::new(WorkerProjectionShutdownGuard { service: Arc::downgrade(&worker_projection), }), @@ -3639,62 +3886,88 @@ impl WorkspaceApi { )) .map_err(|message| Error::Config(message.to_string()))?; } - let expired_external_workdirs = api.store.reconcile_external_workdir_grants_after_restart( - &api.config.workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; + let recovery_api = api.clone(); + let expired_external_workdirs = api + .store + .db_executor() + .run(move || { + recovery_api + .store + .reconcile_external_workdir_grants_after_restart( + &recovery_api.config.workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await?; for workdir_id in expired_external_workdirs { cleanup_external_workdir_attachments(&api, &workdir_id) .await .map_err(|error| error.error)?; } - for grant in api + let grant_api = api.clone(); + let grants = api .store - .list_external_workdir_grants(&api.config.workspace_id)? + .db_executor() + .run(move || { + grant_api + .store + .list_external_workdir_grants(&grant_api.config.workspace_id) + }) + .await?; + for grant in grants .into_iter() .filter(|grant| !matches!(grant.status.as_str(), "revoked" | "expired")) { schedule_external_workdir_expiry(&api, &grant)?; } - recover_workdir_removals(&api)?; + let workdir_recovery_api = api.clone(); + api.host_executors + .host()? + .run(move || recover_workdir_removals(&workdir_recovery_api)) + .await?; recover_runtime_removals(&api).await; - let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let deliveries = api - .store - .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; - for delivery in deliveries { - let Some(job) = api - .store - .get_backend_job(&api.config.workspace_id, &delivery.job_id)? - else { - continue; - }; - let Some(attempt) = api.store.get_backend_job_attempt( - &api.config.workspace_id, - &delivery.job_id, - &delivery.attempt_id, - )? - else { - continue; - }; - if job.state == BackendJobState::Completed - && attempt.state == BackendJobAttemptState::Completed - { - api.deliver_backend_job_result( - &BackendJobResultAcceptance { - job, - attempt, - replayed: true, - }, - &delivery.target, - ); - } - } - api.recover_backend_jobs()?; - if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { - dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); - } - Ok(api) + api.host_executors + .host()? + .run(move || { + let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let deliveries = api + .store + .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; + for delivery in deliveries { + let Some(job) = api + .store + .get_backend_job(&api.config.workspace_id, &delivery.job_id)? + else { + continue; + }; + let Some(attempt) = api.store.get_backend_job_attempt( + &api.config.workspace_id, + &delivery.job_id, + &delivery.attempt_id, + )? + else { + continue; + }; + if job.state == BackendJobState::Completed + && attempt.state == BackendJobAttemptState::Completed + { + api.deliver_backend_job_result( + &BackendJobResultAcceptance { + job, + attempt, + replayed: true, + }, + &delivery.target, + ); + } + } + api.recover_backend_jobs()?; + if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + Ok(api) + }) + .await } pub fn workspace_id(&self) -> &str { @@ -4087,77 +4360,44 @@ impl WorkspaceApi { { return false; } - let Ok(handle) = tokio::runtime::Handle::try_current() else { - tracing::warn!( - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "Backend Job Worker cleanup remains durable because no Tokio runtime is available" - ); + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + tracing::warn!(job_id = %attempt.job_id, "Backend Job cleanup remains durable without a runtime"); return false; }; - let (attempt, claimed) = match self.store.claim_backend_job_worker_cleanup( - &self.config.workspace_id, - &attempt.job_id, - &attempt.attempt_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { - Ok(claim) => claim, - Err(error) => { - tracing::warn!( - %error, - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "failed to claim durable Backend Job Worker cleanup" - ); - return false; - } - }; - if !claimed { - return false; - } - let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed - || attempt.failure_category.as_deref() == Some("invalid_result"); - if waits_for_run_teardown { - let api = self.clone(); - let cleanup_handle = handle.clone(); - handle.spawn(async move { - if let Err(error) = api.wait_for_backend_job_worker_run_teardown(&attempt).await { - tracing::warn!( - %error, - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "Backend Job Worker Run teardown fence failed" - ); - return; - } - api.spawn_claimed_backend_job_worker_cleanup(&cleanup_handle, attempt); - }); - } else { - self.spawn_claimed_backend_job_worker_cleanup(&handle, attempt); - } - true - } - - fn spawn_claimed_backend_job_worker_cleanup( - &self, - handle: &tokio::runtime::Handle, - attempt: BackendJobAttemptRecord, - ) { let api = self.clone(); let job_id = attempt.job_id.clone(); let attempt_id = attempt.attempt_id.clone(); - handle.spawn_blocking(move || { + // Admit before claiming: overload must leave durable Pending/Failed + // ownership recoverable, never strand a Running claim without a job. + let admission = self.host_executors.cleanup().and_then(|executor| { + executor.dispatch(move || { let cleanup = || -> Result<()> { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| Error::Store(error.to_string()))? - .block_on(api.cleanup_backend_job_attempt_worker(attempt)) + let (attempt, claimed) = api.store.claim_backend_job_worker_cleanup( + &api.config.workspace_id, &job_id, &attempt_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; + if !claimed { return Ok(()); } + let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed + || attempt.failure_category.as_deref() == Some("invalid_result"); + runtime.block_on(async { + if waits_for_run_teardown { + api.wait_for_backend_job_worker_run_teardown(&attempt).await?; + } + // Already on the Host owner; removal must not submit to and + // wait on its own lane. All transaction helpers return before I/O. + api.cleanup_backend_job_attempt_worker(attempt).await + }) }; if let Err(error) = cleanup() { tracing::warn!(%error, %job_id, %attempt_id, "Backend Job Worker cleanup failed"); } + }) }); + if let Err(error) = admission { + tracing::warn!(%error, "Backend Job cleanup was not admitted; its durable ownership is unchanged"); + return false; + } + true } async fn wait_for_backend_job_worker_run_teardown( @@ -4292,7 +4532,7 @@ impl WorkspaceApi { // recover that midpoint; without such an operation this remains // a diagnosable retryable failure rather than a direct DB delete. let response = WorkerRemovalService::new(self) - .execute_target_removal( + .execute_target_removal_on_host( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4358,7 +4598,7 @@ impl WorkspaceApi { } let response = WorkerRemovalService::new(self) - .execute_target_removal( + .execute_target_removal_on_host( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4485,6 +4725,7 @@ impl WorkspaceApi { let attempt_id = attempt_id.to_string(); handle.spawn(async move { tokio::time::sleep(delay).await; + let admission = api.host_executors.host().and_then(|executor| executor.dispatch(move || { let Ok(Some(attempt)) = api.store .get_backend_job_attempt(&api.config.workspace_id, &job_id, &attempt_id) @@ -4508,6 +4749,10 @@ impl WorkspaceApi { { api.drain_reserved_backend_jobs(); } + })); + if let Err(error) = admission { + tracing::warn!(%error, "Backend Job timeout was not admitted; its deadline remains durable"); + } }); } @@ -6225,7 +6470,7 @@ async fn attach_server_request_context( } else { let auth = service.auth_api(); match resolve_request_actor( - auth.store.as_ref(), + auth.store.clone(), request.headers(), &auth_public_config(&auth.config).cookie_name, ) @@ -7191,7 +7436,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyRegistrationComplete, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = complete_passkey_registration(&self.auth_api(), request) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || complete_passkey_registration(&api, request)) + .await .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyRegistrationComplete::Status200 { @@ -7222,7 +7472,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyLoginComplete, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = complete_passkey_login(&self.auth_api(), request) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || complete_passkey_login(&api, request)) + .await .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyLoginComplete::Status200 { @@ -7255,7 +7510,11 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - approve_device_login(&self.auth_api(), actor, request) + let api = self.auth_api(); + api.store + .db_executor() + .run(move || approve_device_login(&api, actor, request)) + .await .map_err(ApiError::into_repository_api_error) } @@ -7285,7 +7544,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthLogout, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = logout_browser_session(&self.auth_api(), cookie) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || logout_browser_session(&api, cookie)) + .await .map_err(ApiError::into_repository_api_error)?; Ok(server_api::server_api_responses::AuthLogout::Status200 { body, @@ -7300,9 +7564,12 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.server_api()?; let Some(actor) = context.actor else { + let catalog = api.catalog.clone(); return if api - .catalog - .is_empty() + .store + .db_executor() + .run(move || catalog.is_empty()) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())? { Ok(WorkspaceCatalogListResponse(Vec::new())) @@ -7315,12 +7582,17 @@ impl server_api::ServerApi for ServerApiContractService { )) }; }; + let catalog = api.catalog.clone(); let workspaces = api - .catalog - .list( - &actor.account_id, - query.limit.unwrap_or(100).try_into().unwrap_or(200), - ) + .store + .db_executor() + .run(move || { + catalog.list( + &actor.account_id, + query.limit.unwrap_or(100).try_into().unwrap_or(200), + ) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; Ok(WorkspaceCatalogListResponse( workspaces.into_iter().map(workspace_summary).collect(), @@ -7341,9 +7613,12 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; + let catalog = api.catalog.clone(); let created = api - .catalog - .create(request, actor.account_id) + .store + .db_executor() + .run(move || catalog.create(request, actor.account_id)) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; api.router_for_workspace(&created.workspace.workspace_id) .await @@ -7388,10 +7663,14 @@ impl server_api::ServerApi for ServerApiContractService { ) })?; let admission_for_workspace = api.admission_for_workspace(&workspace_id).await; - let _mutation_guard = admission_for_workspace.lock_deletion().await; + let mutation_guard = Arc::new(admission_for_workspace.lock_deletion().await); + let account_id = actor.account_id.clone(); + let operation_id = request.operation_id.clone(); let existing = api - .store - .workspace_deletion_operation(&actor.account_id, &request.operation_id) + .execute_store_stage(move |store| { + store.workspace_deletion_operation(&account_id, &operation_id) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; if existing.is_none() { let preflight = api @@ -7407,9 +7686,18 @@ impl server_api::ServerApi for ServerApiContractService { )); } } + let account_id = actor.account_id.clone(); + let id = workspace_id.clone(); + let reserve_request = request.clone(); + let reserve_guard = mutation_guard.clone(); let reservation = api - .store - .reserve_workspace_deletion(&actor.account_id, &workspace_id, &request) + .execute_store_stage(move |store| { + // Keep deletion closed until the accepted reservation really finishes, + // even when the HTTP task drops its original guard. + let _guard = reserve_guard; + store.reserve_workspace_deletion(&account_id, &id, &reserve_request) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; if let Some(handle) = api.hook_handles.lock().await.remove(&workspace_id) { handle.abort(); @@ -7449,17 +7737,19 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - api.store - .workspace_deletion_operation(&actor.account_id, &operation_id) - .map_err(|error| ApiError::from(error).into_repository_api_error())? - .ok_or_else(|| { - server_api::RepositoryApiError::new( - StatusCode::NOT_FOUND.as_u16(), - "Not Found", - "Workspace deletion operation not found", - Vec::new(), - ) - }) + api.execute_store_stage(move |store| { + store.workspace_deletion_operation(&actor.account_id, &operation_id) + }) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? + .ok_or_else(|| { + server_api::RepositoryApiError::new( + StatusCode::NOT_FOUND.as_u16(), + "Not Found", + "Workspace deletion operation not found", + Vec::new(), + ) + }) } async fn workspace_current( @@ -7494,13 +7784,20 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::WorkspaceRuntimeConfig, server_api::RepositoryApiError, > { - workspace_runtime_config_result( - self.workspace_api()?, - context, - &workspace_id, - query, - if_none_match, - ) + let api = self.workspace_api()?.clone(); + api.store + .db_executor() + .run(move || { + Ok::<_, Error>(workspace_runtime_config_result( + &api, + context, + &workspace_id, + query, + if_none_match, + )) + }) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? } async fn workspace_metadata_settings( @@ -9393,7 +9690,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; restore_runtime_worker_with_context(api, runtime_id, worker_id, operation_context, request) .await .map(|Json(response)| response) @@ -9411,7 +9708,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; scoped_restore_runtime_worker_with_context( @@ -9438,10 +9735,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, true) @@ -9458,10 +9756,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, false) @@ -9498,7 +9797,7 @@ impl server_api::ServerApi for ServerApiContractService { server_api::RepositoryApiError, > { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; execute_runtime_cleanup_with_context(&api, &runtime_id, request, &operation_context) @@ -9515,8 +9814,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -9537,10 +9837,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -9700,8 +10001,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -9723,10 +10025,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -9747,8 +10050,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -9770,10 +10074,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -10588,23 +10893,28 @@ impl server_api::ServerApi for ServerApiContractService { query: server_api::TicketAssignmentsQuery, ) -> std::result::Result { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; + let api = self.workspace_api()?.clone(); api.store - .page_ticket_assignments( - &workspace_id, - &ticket_id, - query.after_assignment_id.as_deref(), - query.limit.unwrap_or(30), - ) - .map(server_api::TicketAssignmentsPage) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error) + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + api.store + .page_ticket_assignments( + &workspace_id, + &ticket_id, + query.after_assignment_id.as_deref(), + query.limit.unwrap_or(30), + ) + .map(server_api::TicketAssignmentsPage) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error) + }) + .await } async fn ticket_assignments_page_browser( &self, @@ -10627,24 +10937,29 @@ impl server_api::ServerApi for ServerApiContractService { server_api::TicketAssignmentDiagnosticsPage, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; + let api = self.workspace_api()?.clone(); api.store - .page_ticket_assignment_diagnostics( - &workspace_id, - &ticket_id, - &query.assignment_id, - query.after_input_event_id.as_deref(), - query.after_start_operation_id.as_deref(), - query.limit.unwrap_or(30), - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error) + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + api.store + .page_ticket_assignment_diagnostics( + &workspace_id, + &ticket_id, + &query.assignment_id, + query.after_input_event_id.as_deref(), + query.after_start_operation_id.as_deref(), + query.limit.unwrap_or(30), + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error) + }) + .await } async fn ticket_assignment_diagnostics_browser( &self, @@ -10669,26 +10984,74 @@ impl server_api::ServerApi for ServerApiContractService { server_api::TicketAssignmentAnswerResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let (actor, source) = - authenticated_assignment_source(api, &contract_request_headers(&context)?, &context) + let api = self.workspace_api()?.clone(); + // Runtime discovery may perform remote I/O. Preserve the live singleton + // check on the finite Host lane; the answer's source/binding authority + // is checked again under its IMMEDIATE transaction on the DB lane. + if context.runtime_source.is_some() { + let source_api = api.clone(); + let source_context = context.clone(); + let source_workspace_id = workspace_id.clone(); + api.host_executors + .host() + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)? + .run(move || { + validate_workspace_scope(&source_api, &source_workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + authenticated_assignment_source( + &source_api, + &contract_request_headers(&source_context)?, + &source_context, + ) + .map(|_| ()) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error) + }) + .await?; + } + let (receipt, notification) = api + .store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let (actor, source) = assignment_source_from_authenticated_context( + &contract_request_headers(&context)?, + &context, + ) .map_err(ApiError::from) .map_err(ApiError::into_repository_api_error)?; - let receipt = api - .store - .answer_ticket_assignment(&workspace_id, &ticket_id, &request.0, &source, &actor) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - // The entire event/audit/receipt transaction has committed. Response loss - // retries the saved receipt; delivery scan deduplicates accepted inputs. - if let Some(orchestrator) = find_workspace_orchestrator(api) { - dispatch_orchestrator_assignment_attention(api, &orchestrator.worker); + let receipt = api + .store + .answer_ticket_assignment( + &workspace_id, + &ticket_id, + &request.0, + &source, + &actor, + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + // This admitted SQL job owns source/fence/CAS/event/audit/receipt + // and commit. Admit its post-commit Host continuation here, not in + // the cancellable waiter. It inherits the request lifecycle lease; + // Runtime/network dispatch never occupies the DB worker. + Ok::<_, server_api::RepositoryApiError>(( + receipt, + enqueue_orchestrator_assignment_attention(&api), + )) + }) + .await?; + if let Some(notification) = notification { + // Preserve normal best-effort notification completion for a live + // response. Losing this receiver cannot cancel the accepted job. + let _ = notification.await; } Ok(server_api::TicketAssignmentAnswerResponse(receipt)) } @@ -10714,23 +11077,28 @@ impl server_api::ServerApi for ServerApiContractService { query: server_api::TicketAssignmentTrackingQuery, ) -> std::result::Result { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let values = if let Some(assignment_id) = query.assignment_id { - api.store - .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) - .map(|value| value.into_iter().collect::>()) - } else { - api.authority.ticket_assignment_tracking(&ticket_id) - } - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - Ok(server_api::TicketAssignmentTrackingList(values)) + let api = self.workspace_api()?.clone(); + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let values = if let Some(assignment_id) = query.assignment_id { + api.store + .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) + .map(|value| value.into_iter().collect::>()) + } else { + api.authority.ticket_assignment_tracking(&ticket_id) + } + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + Ok(server_api::TicketAssignmentTrackingList(values)) + }) + .await } async fn ticket_assignment_tracking_update( @@ -11557,21 +11925,30 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirCatalogResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) + let api = self.workspace_api()?.clone(); + let headers = current_worker_contract_headers(&context)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host() + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &workspace_id, + &headers, + )?) + }) + .await .map_err(ApiError::into_repository_api_error)?; - let worker = current_worker_identity( - api, - &workspace_id, - ¤t_worker_contract_headers(&context)?, - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - { - let session_lock = current_worker_session_lock(api, &worker); - let _guard = session_lock.lock().await; - list_current_worker_workdir_catalog_for_worker(api, &worker, query) - } + let session_lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(session_lock.lock_owned().await); + run_worker_workdir_db_stage_locked(&api, &guard, move |api| { + list_current_worker_workdir_catalog_for_worker(&api, &worker, query) + }) + .await .map_err(ApiError::into_repository_api_error) } @@ -11584,20 +11961,31 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirAttachmentListResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) + let api = self.workspace_api()?.clone(); + let headers = current_worker_contract_headers(&context)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host() + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &workspace_id, + &headers, + )?) + }) + .await .map_err(ApiError::into_repository_api_error)?; - let worker = current_worker_identity( - api, - &workspace_id, - ¤t_worker_contract_headers(&context)?, - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let session_lock = current_worker_session_lock(api, &worker); - let _session_guard = session_lock.lock().await; - list_current_worker_workdir_attachments(api, &worker, query) - .map_err(ApiError::into_repository_api_error) + let session_lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(session_lock.lock_owned().await); + run_worker_workdir_db_stage_locked(&api, &guard, move |api| { + list_current_worker_workdir_attachments(&api, &worker, query) + }) + .await + .map_err(ApiError::into_repository_api_error) } async fn current_worker_workdir_attach( @@ -12299,8 +12687,13 @@ async fn scoped_list_flows( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult>> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) + }) + .await } async fn scoped_put_flow( @@ -12308,28 +12701,34 @@ async fn scoped_put_flow( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let definition = flow::compile_flow_source(&request.content).map_err(|error| { - Error::InvalidInput(format!( - "invalid Flow source: {}", - serde_json::to_string(&error.diagnostics) - .unwrap_or_else(|_| "diagnostics unavailable".to_string()) - )) - })?; - let expected_path = format!("flows/{}.dcdl", definition.name); - if request.path != expected_path { - return Err( - Error::InvalidInput(format!("Flow source path must be `{expected_path}`")).into(), - ); - } - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); - Ok(Json(api.store.put_flow_source_for_kind( - &path.workspace_id, - FlowSourceKind::Workspace, - &request.path, - &request.content, - &now, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let definition = flow::compile_flow_source(&request.content).map_err(|error| { + Error::InvalidInput(format!( + "invalid Flow source: {}", + serde_json::to_string(&error.diagnostics) + .unwrap_or_else(|_| "diagnostics unavailable".to_string()) + )) + })?; + let expected_path = format!("flows/{}.dcdl", definition.name); + if request.path != expected_path { + return Err(Error::InvalidInput(format!( + "Flow source path must be `{expected_path}`" + )) + .into()); + } + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + Ok(Json(api.store.put_flow_source_for_kind( + &path.workspace_id, + FlowSourceKind::Workspace, + &request.path, + &request.content, + &now, + )?)) + }) + .await } async fn scoped_resolve_flow_source( @@ -12337,65 +12736,79 @@ async fn scoped_resolve_flow_source( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = match &request.selector { - flow::FlowSelector::Builtin { slug } => { - let builtin = flow::builtin_flow_source(slug) - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let definition = builtin.compile().map_err(|error| { - Error::Store(format!( - "compile built-in Flow {slug:?}: {:?}", - error.diagnostics - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: format!("builtin:{slug}"), - content_digest: definition.content_digest.clone(), - definition, - } - } - flow::FlowSelector::Workspace { slug } => { - let source = api - .store - .get_flow_source_by_name(&path.workspace_id, FlowSourceKind::Workspace, slug)? - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let content = api - .store - .get_flow_source_content( - &path.workspace_id, - &source.flow_id, - &source.content_digest, - )? - .ok_or_else(|| { - Error::Store(format!( - "resolved Flow content {}@{} is missing", - source.flow_id, source.content_digest - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: source.flow_id, - content_digest: content.content_digest, - definition: content.definition, - } - } - }; - Ok(Json(resolved)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let resolved = match &request.selector { + flow::FlowSelector::Builtin { slug } => { + let builtin = flow::builtin_flow_source(slug) + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let definition = builtin.compile().map_err(|error| { + Error::Store(format!( + "compile built-in Flow {slug:?}: {:?}", + error.diagnostics + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: format!("builtin:{slug}"), + content_digest: definition.content_digest.clone(), + definition, + } + } + flow::FlowSelector::Workspace { slug } => { + let source = api + .store + .get_flow_source_by_name( + &path.workspace_id, + FlowSourceKind::Workspace, + slug, + )? + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let content = api + .store + .get_flow_source_content( + &path.workspace_id, + &source.flow_id, + &source.content_digest, + )? + .ok_or_else(|| { + Error::Store(format!( + "resolved Flow content {}@{} is missing", + source.flow_id, source.content_digest + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: source.flow_id, + content_digest: content.content_digest, + definition: content.definition, + } + } + }; + Ok(Json(resolved)) + }) + .await } async fn scoped_get_flow( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = api - .store - .get_flow_source(&path.workspace_id, &path.flow_id)? - .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; - Ok(Json(source)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = api + .store + .get_flow_source(&path.workspace_id, &path.flow_id)? + .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; + Ok(Json(source)) + }) + .await } async fn scoped_get_workspace_settings( @@ -12466,8 +12879,13 @@ async fn scoped_get_workspace_signing_identity( "Workspace public identity access", ) .await?; - let identity = api.signing_identities.get_validated(&path.workspace_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) + api.store + .db_executor() + .run(move || { + let identity = api.signing_identities.get_validated(&path.workspace_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) + }) + .await } async fn scoped_provision_workspace_signing_identity( @@ -12482,10 +12900,15 @@ async fn scoped_provision_workspace_signing_identity( "Workspace signing identity provisioning", ) .await?; - let identity = api - .signing_identities - .provision_existing(&path.workspace_id, &actor.account_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) + api.store + .db_executor() + .run(move || { + let identity = api + .signing_identities + .provision_existing(&path.workspace_id, &actor.account_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) + }) + .await } fn project_workspace_signing_identity( @@ -12563,12 +12986,19 @@ async fn scoped_get_workspace_config_history( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let snapshot = api - .config_store - .load_workspace_config_history(&path.workspace_id, &path.content_digest)? - .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.content_digest.clone())))?; - Ok(Json(config_tree_snapshot_to_api(snapshot))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let snapshot = api + .config_store + .load_workspace_config_history(&path.workspace_id, &path.content_digest)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId(path.content_digest.clone())) + })?; + Ok(Json(config_tree_snapshot_to_api(snapshot))) + }) + .await } #[derive(Debug, Deserialize)] @@ -12581,34 +13011,44 @@ async fn scoped_get_prompt_projection( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidInput( - "Workspace config is not initialized".to_string(), - )) - })?; - let projection = api - .prompt_projection_cache - .resolve(&path.workspace_id, &state)?; - Ok(Json(projection.as_ref().clone())) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidInput( + "Workspace config is not initialized".to_string(), + )) + })?; + let projection = api + .prompt_projection_cache + .resolve(&path.workspace_id, &state)?; + Ok(Json(projection.as_ref().clone())) + }) + .await } async fn scoped_get_workspace_memory_settings( State(api): State, AxumPath(workspace_id): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .get_workspace_memory_settings(&workspace_id) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .get_workspace_memory_settings(&workspace_id) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) + }) + .await } async fn scoped_update_workspace_memory_settings( @@ -12616,19 +13056,24 @@ async fn scoped_update_workspace_memory_settings( AxumPath(workspace_id): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .update_workspace_memory_settings( - &workspace_id, - &request.expected_language, - &request.language, - ) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .update_workspace_memory_settings( + &workspace_id, + &request.expected_language, + &request.language, + ) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) + }) + .await } async fn require_workspace_owner( @@ -13139,41 +13584,51 @@ async fn scoped_get_workspace_config_tree( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { - snapshot: ConfigTreeSnapshot::empty(), - contract: config_source::ToolchainContract::new( - config_source::DEFAULT_SCHEMA_VERSION, - Vec::new(), - config_source::DEFAULT_IMPORT_POLICY_VERSION, - ), - projection_digest: config_source::digest_bytes(b"[]"), - }); - Ok(Json(workspace_config_state_to_api(state))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { + snapshot: ConfigTreeSnapshot::empty(), + contract: config_source::ToolchainContract::new( + config_source::DEFAULT_SCHEMA_VERSION, + Vec::new(), + config_source::DEFAULT_IMPORT_POLICY_VERSION, + ), + projection_digest: config_source::digest_bytes(b"[]"), + }); + Ok(Json(workspace_config_state_to_api(state))) + }) + .await } async fn scoped_get_workspace_config_entry( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let virtual_path = config_source::VirtualPath::parse(&path.path) - .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - let entry = state - .snapshot - .get(&virtual_path) - .cloned() - .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; - Ok(Json(config_entry_to_api(entry))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let virtual_path = config_source::VirtualPath::parse(&path.path) + .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + let entry = state + .snapshot + .get(&virtual_path) + .cloned() + .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; + Ok(Json(config_entry_to_api(entry))) + }) + .await } async fn scoped_commit_workspace_config_tree( @@ -13181,13 +13636,18 @@ async fn scoped_commit_workspace_config_tree( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - let request = config_commit_request_from_api(request)?; - validate_workspace_scope(&api, &path.workspace_id)?; - let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; - Ok(( - StatusCode::CREATED, - Json(workspace_config_state_to_api(state)), - )) + api.store + .db_executor() + .run(move || { + let request = config_commit_request_from_api(request)?; + validate_workspace_scope(&api, &path.workspace_id)?; + let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; + Ok(( + StatusCode::CREATED, + Json(workspace_config_state_to_api(state)), + )) + }) + .await } /// Shared canonical commit and activation boundary for UI and logical config operations. @@ -13251,20 +13711,25 @@ async fn scoped_get_profile_settings( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - Ok(Json( - crate::profile_settings::project_profiles_from_workspace_config( - &path.workspace_id, - &state, - )? - .settings, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + Ok(Json( + crate::profile_settings::project_profiles_from_workspace_config( + &path.workspace_id, + &state, + )? + .settings, + )) + }) + .await } async fn scoped_list_tickets( @@ -13280,8 +13745,13 @@ async fn scoped_get_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - browser_ticket_detail(&api, &path.id) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_query_tickets( @@ -13289,8 +13759,13 @@ async fn scoped_query_tickets( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_tickets(query)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_tickets(query)?)) + }) + .await } async fn scoped_show_ticket( @@ -13298,11 +13773,16 @@ async fn scoped_show_ticket( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let backend = browser_ticket_backend(&api)?; - Ok(Json(api.authority.read_ticket_detail_with_backend( - &path.id, query, &backend, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let backend = browser_ticket_backend(&api)?; + Ok(Json(api.authority.read_ticket_detail_with_backend( + &path.id, query, &backend, + )?)) + }) + .await } fn parse_ticket_assignment_role(role: &str) -> ApiResult { @@ -13378,19 +13858,24 @@ async fn scoped_list_ticket_assignments( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let ticket = api.authority.ticket(&path.id)?; - let assignments = api - .store - .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; - Ok(Json(server_api::TicketRoleAssignmentsResponse { - workspace_id: path.workspace_id, - ticket_id: ticket.id, - assignments: assignments - .into_iter() - .map(project_ticket_role_assignment_record) - .collect(), - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let ticket = api.authority.ticket(&path.id)?; + let assignments = api + .store + .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; + Ok(Json(server_api::TicketRoleAssignmentsResponse { + workspace_id: path.workspace_id, + ticket_id: ticket.id, + assignments: assignments + .into_iter() + .map(project_ticket_role_assignment_record) + .collect(), + })) + }) + .await } async fn compensate_manual_ticket_worker_binding( @@ -13529,6 +14014,31 @@ async fn start_manual_ticket_worker_assignment( operation_id: &str, selected: &[server_api::TicketWorkerAttachmentBinding], delegation: Option<&TicketStartDelegation>, +) -> ApiResult { + let api = api.clone(); + let record = record.clone(); + let operation_id = operation_id.to_owned(); + let selected = selected.to_vec(); + let delegation = delegation.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + start_manual_ticket_worker_assignment_on_host( + &api, + &record, + &operation_id, + &selected, + delegation.as_ref(), + ) + .await + }) + .await +} + +async fn start_manual_ticket_worker_assignment_on_host( + api: &WorkspaceApi, + record: &TicketRoleAssignmentRecord, + operation_id: &str, + selected: &[server_api::TicketWorkerAttachmentBinding], + delegation: Option<&TicketStartDelegation>, ) -> ApiResult { let TicketAssignmentPrincipal::Worker { runtime_id, @@ -13945,13 +14455,35 @@ fn authenticated_assignment_source( String, crate::store::TicketOrchestratorAssignmentSourceContext, )> { - if let Some(source) = authenticated_ticket_worker_source(headers, context)? { + let result = assignment_source_from_authenticated_context(headers, context)?; + if let crate::store::TicketOrchestratorAssignmentSourceContext::Runtime { + runtime_id, + worker_id, + .. + } = &result.1 + { + let source = RuntimeWorkerRef::new(runtime_id, worker_id); if !find_workspace_orchestrator(api).is_some_and(|worker| worker.worker == source) { return Err(Error::WorkspacePermissionDenied( "Ticket assignment mutations require the registered current Workspace Orchestrator" .into(), )); } + } + Ok(result) +} + +// Attribution only: Backend authenticated this context, never body input. This +// performs no Runtime lookup. Store mutation helpers fence the current singleton +// and live authentication binding in the same transaction as CAS/audit/receipt. +fn assignment_source_from_authenticated_context( + headers: &HeaderMap, + context: &server_api::ServerRequestContext, +) -> Result<( + String, + crate::store::TicketOrchestratorAssignmentSourceContext, +)> { + if let Some(source) = authenticated_ticket_worker_source(headers, context)? { let authentication_binding_id = context .runtime_source .as_ref() @@ -13997,7 +14529,13 @@ async fn scoped_set_ticket_assignment_with_context( request: Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let (author, source) = authenticated_assignment_source(&api, &headers, context)?; + let source_api = api.clone(); + let context = context.clone(); + let (author, source) = api + .host_executors + .host()? + .run(move || authenticated_assignment_source(&source_api, &headers, &context)) + .await?; apply_set_ticket_assignment(State(api), path, request, author, Some(source)).await } @@ -14008,46 +14546,83 @@ async fn apply_set_ticket_assignment( assigned_by: String, source: Option, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = require_ticket_assignment_value("operation_id", request.operation_id)?; - let expected_assignment_id = request - .expected_assignment_id - .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) - .transpose()?; - let principal = internal_ticket_assignment_principal(request.principal)?; - let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let record = TicketRoleAssignmentRecord { - workspace_id: workspace_id.clone(), - ticket_id: ticket.id.clone(), - assignment_id: new_id("tasg"), - role, - principal, - assigned_by, - assigned_at, - }; - let assignment = match role { + let prep_api = api.clone(); + let prep_workspace = workspace_id.clone(); + let (ticket, role, operation_id, expected_assignment_id, record, workdir_bindings) = api + .store + .db_executor() + .run(move || { + let api = &prep_api; + let workspace_id = prep_workspace; + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = + require_ticket_assignment_value("operation_id", request.operation_id)?; + let expected_assignment_id = request + .expected_assignment_id + .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) + .transpose()?; + let principal = internal_ticket_assignment_principal(request.principal)?; + let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let record = TicketRoleAssignmentRecord { + workspace_id: workspace_id.clone(), + ticket_id: ticket.id.clone(), + assignment_id: new_id("tasg"), + role, + principal, + assigned_by, + assigned_at, + }; + Ok::<_, ApiError>(( + ticket, + role, + operation_id, + expected_assignment_id, + record, + request.workdir_bindings, + )) + }) + .await?; + let (assignment, notification) = match role { TicketAssignmentRole::Orchestrator => { - if let Some(source) = source.as_ref() { - api.store.set_current_ticket_orchestrator_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - source, - )? - } else { - // Only the cfg(test) direct-handler adapter lacks a Backend source. - api.store.set_current_ticket_role_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - )? - } + let store = api.store.clone(); + let notification_api = api.clone(); + let record = record.clone(); + let expected = expected_assignment_id.clone(); + let operation_id = operation_id.clone(); + api.store + .db_executor() + .run(move || { + let expected_assignment_id = expected; + let assignment = if let Some(source) = source.as_ref() { + store.set_current_ticket_orchestrator_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + source, + ) + } else { + // Only the cfg(test) direct-handler adapter lacks a Backend source. + store.set_current_ticket_role_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + ) + }?; + // Dispatch from the accepted SQL owner after commit, even + // when its waiter has gone. The response still observes the + // same best-effort notification completion as before offload. + Ok::<_, Error>(( + assignment, + enqueue_orchestrator_assignment_attention(¬ification_api), + )) + }) + .await? } TicketAssignmentRole::Worker => { if expected_assignment_id.is_some() { @@ -14061,14 +14636,17 @@ async fn apply_set_ticket_assignment( source.as_ref(), request.orchestrator_assignment_id.as_deref(), )?; - start_manual_ticket_worker_assignment( - &api, - &record, - &operation_id, - &request.workdir_bindings, - delegation.as_ref(), + ( + start_manual_ticket_worker_assignment( + &api, + &record, + &operation_id, + &workdir_bindings, + delegation.as_ref(), + ) + .await?, + None, ) - .await? } TicketAssignmentRole::Owner | TicketAssignmentRole::Contributor => { return Err(Error::TicketAssignmentConflict( @@ -14078,12 +14656,8 @@ async fn apply_set_ticket_assignment( .into()); } }; - if role == TicketAssignmentRole::Orchestrator - && let Some(orchestrator) = find_workspace_orchestrator(&api) - { - // The Assign row/event/receipt has committed. Delivery failure cannot - // roll back delegation; startup/restore/idle use this same durable scan. - dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + if let Some(notification) = notification { + let _ = notification.await; } Ok(Json(server_api::TicketRoleAssignmentMutationResponse { workspace_id, @@ -14109,7 +14683,13 @@ async fn scoped_clear_ticket_assignment_with_context( query: Query, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let (author, source) = authenticated_assignment_source(&api, &headers, context)?; + let source_api = api.clone(); + let context = context.clone(); + let (author, source) = api + .host_executors + .host()? + .run(move || authenticated_assignment_source(&source_api, &headers, &context)) + .await?; apply_clear_ticket_assignment(State(api), path, query, author, Some(source)).await } @@ -14120,65 +14700,70 @@ async fn apply_clear_ticket_assignment( author: String, source: Option, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = query - .operation_id - .map(|value| require_ticket_assignment_value("operation_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) - })?; - let assignment_id = query - .assignment_id - .map(|value| require_ticket_assignment_value("assignment_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) - })?; - let event_id = new_id("tasev"); - let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let reason = Some("role assignment removed from Ticket detail"); - let cleared = if role == TicketAssignmentRole::Orchestrator - && let Some(source) = source.as_ref() - { - api.store.clear_current_ticket_orchestrator_assignment( - &workspace_id, - &ticket.id, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - source, - )? - } else { - api.store.clear_current_ticket_role_assignment( - &workspace_id, - &ticket.id, - role, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - )? - }; - if !cleared { - return Err(Error::TicketAssignmentConflict(format!( - "assignment `{assignment_id}` is not current for role `{}`", - role.as_str() - )) - .into()); - } - Ok(Json(server_api::TicketRoleAssignmentMutationResponse { - workspace_id, - ticket_id: ticket.id, - assignment: None, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = query + .operation_id + .map(|value| require_ticket_assignment_value("operation_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) + })?; + let assignment_id = query + .assignment_id + .map(|value| require_ticket_assignment_value("assignment_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) + })?; + let event_id = new_id("tasev"); + let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let reason = Some("role assignment removed from Ticket detail"); + let cleared = if role == TicketAssignmentRole::Orchestrator + && let Some(source) = source.as_ref() + { + api.store.clear_current_ticket_orchestrator_assignment( + &workspace_id, + &ticket.id, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + source, + )? + } else { + api.store.clear_current_ticket_role_assignment( + &workspace_id, + &ticket.id, + role, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + )? + }; + if !cleared { + return Err(Error::TicketAssignmentConflict(format!( + "assignment `{assignment_id}` is not current for role `{}`", + role.as_str() + )) + .into()); + } + Ok(Json(server_api::TicketRoleAssignmentMutationResponse { + workspace_id, + ticket_id: ticket.id, + assignment: None, + })) + }) + .await } async fn scoped_cancel_ticket_implementation( @@ -14697,49 +15282,54 @@ async fn scoped_edit_ticket_item( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let mut targets = request.target.map(|target| match target { - server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { - targets: targets - .into_iter() - .map(|target| ticket::TicketTarget { - repository_key: target.repository_key, - ref_selector: target.ref_selector, - access: target.access, - }) - .collect(), - }, - server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, - }); - if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { - validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; - } - browser_ticket_backend(&api)? - .edit_item( - TicketIdOrSlug::Id(path.id.clone()), - TicketItemEdit { - title: request.title, - body: request.body.map(MarkdownText::new), - body_replacement: match (request.old_string, request.new_string) { - (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { - old_string, - new_string, - replace_all: request.replace_all, - }), - (None, None) => None, - _ => { - return Err(settings_bad_request( - "invalid_ticket_edit_replacement", - "old_string and new_string must be provided together", - )); - } + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let mut targets = request.target.map(|target| match target { + server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { + targets: targets + .into_iter() + .map(|target| ticket::TicketTarget { + repository_key: target.repository_key, + ref_selector: target.ref_selector, + access: target.access, + }) + .collect(), }, - targets, - author: request.author, - }, - ) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) + server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, + }); + if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { + validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; + } + browser_ticket_backend(&api)? + .edit_item( + TicketIdOrSlug::Id(path.id.clone()), + TicketItemEdit { + title: request.title, + body: request.body.map(MarkdownText::new), + body_replacement: match (request.old_string, request.new_string) { + (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { + old_string, + new_string, + replace_all: request.replace_all, + }), + (None, None) => None, + _ => { + return Err(settings_bad_request( + "invalid_ticket_edit_replacement", + "old_string and new_string must be provided together", + )); + } + }, + targets, + author: request.author, + }, + ) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_append_ticket_event( @@ -14747,21 +15337,26 @@ async fn scoped_append_ticket_event( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = match request.role { - server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, - server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, - server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, - server_api::BrowserTicketThreadRole::ImplementationReport => { - TicketEventKind::ImplementationReport - } - }; - let mut event = NewTicketEvent::new(role, request.body); - event.author = request.author; - browser_ticket_backend(&api)? - .add_event(TicketIdOrSlug::Id(path.id.clone()), event) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let role = match request.role { + server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, + server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, + server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, + server_api::BrowserTicketThreadRole::ImplementationReport => { + TicketEventKind::ImplementationReport + } + }; + let mut event = NewTicketEvent::new(role, request.body); + event.author = request.author; + browser_ticket_backend(&api)? + .add_event(TicketIdOrSlug::Id(path.id.clone()), event) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_mark_ticket_ready_from_browser( @@ -14940,17 +15535,24 @@ fn schedule_ticket_item_check( } }; let api = api.clone(); - tokio::task::spawn_blocking(move || { - if let Err(error) = api.dispatch_backend_job_reservation(reservation) { - tracing::warn!( - job_id = %request.job_id, - ticket_input_digest = ?request.input_digest(), - %error, - "asynchronous Ticket item checker dispatch failed" - ); - api.drain_reserved_backend_jobs(); - } + let admission = api.host_executors.host().and_then(|executor| { + executor.dispatch(move || { + if let Err(error) = api.dispatch_backend_job_reservation(reservation) { + tracing::warn!( + job_id = %request.job_id, + ticket_input_digest = ?request.input_digest(), + %error, + "asynchronous Ticket item checker dispatch failed" + ); + if !matches!(error, Error::DbOperationOutcomeUnknown) { + api.drain_reserved_backend_jobs(); + } + } + }) }); + if let Err(error) = admission { + tracing::warn!(%error, "Ticket checker was not admitted; its reservation remains durable"); + } } async fn execute_ticket_rest_operation( @@ -14986,6 +15588,13 @@ async fn execute_ticket_rest_operation_with_context( .await?; return Ok(TicketBackendOperationResult::Ticket(updated)); } + let job_api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let context = context.cloned(); + api.host_executors.host()?.run(move || { + let api = &job_api; + let workspace_id = workspace_id.as_str(); + let context = context.as_ref(); validate_ticket_operation_repository_keys(api, workspace_id, &operation)?; let mut backend = SqliteTicketBackend::open_verified( api.config.database_path.clone(), @@ -15183,6 +15792,7 @@ async fn execute_ticket_rest_operation_with_context( } } Ok(result) + }).await } #[cfg(test)] @@ -16145,6 +16755,8 @@ async fn scoped_list_merge_requests( AxumPath(workspace_id): AxumPath, Query(query): Query, ) -> ApiResult> { + api.host_executors.host()?.run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; require_workspace_access(&workspace_id, &api)?; let ticket_id = query @@ -16253,6 +16865,8 @@ async fn scoped_list_merge_requests( items, next_cursor: page.next_cursor, })) + + }).await } fn merge_ref_diagnostic(error: ApiError) -> MergeRequestRefDiagnostic { @@ -16358,159 +16972,169 @@ async fn scoped_show_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let store = merge_request_store(&api, &workspace_id)?; - let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let integrated_refs = integrated_merge_refs(&mr); - mr.thread = store.thread_page_by_id( - &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - let ticket_id = mr.ticket_ids.first().ok_or_else(|| { - Error::MergeRequest(merge_request::MergeRequestError::Conflict( - "Merge Request has no linked Ticket".into(), - )) - })?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; - let (source, target) = if let Some(refs) = integrated_refs { - refs - } else { - let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { - (Some(selector), Some(assignment)) => { - match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - } - } - (Some(_), None) => unknown_merge_ref( - "source_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the source ref", - ), - (None, _) => server_api::MergeRequestRefResponse { - status: "requires_repair".into(), - resolved_ref: None, - observed_at: Utc::now().to_rfc3339(), - diagnostic: None, - }, - }; - let target = match assignment.as_ref() { - Some(assignment) => match observe_published_merge_ref( - &api, + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let store = merge_request_store(&api, &workspace_id)?; + let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let integrated_refs = integrated_merge_refs(&mr); + mr.thread = store.thread_page_by_id( &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - }, - None => unknown_merge_ref( - "target_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the target ref", - ), - }; - (source, target) - }; - let linked_tickets = mr - .ticket_ids - .iter() - .map(|ticket_id| { - Ok(server_api::MergeRequestLinkedTicketResponse { - ticket_id: ticket_id.clone(), - key: api.store.resource_key( - &workspace_id, - WorkspaceResourceKind::Ticket, - ticket_id, - )?, - }) + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + let ticket_id = mr.ticket_ids.first().ok_or_else(|| { + Error::MergeRequest(merge_request::MergeRequestError::Conflict( + "Merge Request has no linked Ticket".into(), + )) + })?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; + let (source, target) = if let Some(refs) = integrated_refs { + refs + } else { + let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { + (Some(selector), Some(assignment)) => { + match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + } + } + (Some(_), None) => unknown_merge_ref( + "source_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the source ref", + ), + (None, _) => server_api::MergeRequestRefResponse { + status: "requires_repair".into(), + resolved_ref: None, + observed_at: Utc::now().to_rfc3339(), + diagnostic: None, + }, + }; + let target = match assignment.as_ref() { + Some(assignment) => match observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + }, + None => unknown_merge_ref( + "target_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the target ref", + ), + }; + (source, target) + }; + let linked_tickets = mr + .ticket_ids + .iter() + .map(|ticket_id| { + Ok(server_api::MergeRequestLinkedTicketResponse { + ticket_id: ticket_id.clone(), + key: api.store.resource_key( + &workspace_id, + WorkspaceResourceKind::Ticket, + ticket_id, + )?, + }) + }) + .collect::>>()?; + let merge_request = public_merge_request(&api, &workspace_id, mr)?; + Ok(Json(server_api::MergeRequestDetailResponse { + merge_request, + source, + target, + linked_tickets, + })) }) - .collect::>>()?; - let merge_request = public_merge_request(&api, &workspace_id, mr)?; - Ok(Json(server_api::MergeRequestDetailResponse { - merge_request, - source, - target, - linked_tickets, - })) + .await } async fn scoped_merge_request_readiness( State(api): State, AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; - let (current_subject_ref, source_blocker) = - if mr.state == merge_request::MergeRequestState::Merged { - ( - mr.merged_result() - .ok() - .map(|merge| merge.approved_source_ref.clone()), - None, - ) - } else { - match (mr.selector_from.as_deref(), assignment) { - (Some(selector), Some(assignment)) => match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => (Some(observation.resolved_ref), None), - Err(error) => { - let diagnostic = merge_ref_diagnostic(error); - let blocker = source_ref_readiness_blocker(&diagnostic.code); - (None, Some(blocker.to_string())) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; + let (current_subject_ref, source_blocker) = + if mr.state == merge_request::MergeRequestState::Merged { + ( + mr.merged_result() + .ok() + .map(|merge| merge.approved_source_ref.clone()), + None, + ) + } else { + match (mr.selector_from.as_deref(), assignment) { + (Some(selector), Some(assignment)) => match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => (Some(observation.resolved_ref), None), + Err(error) => { + let diagnostic = merge_ref_diagnostic(error); + let blocker = source_ref_readiness_blocker(&diagnostic.code); + (None, Some(blocker.to_string())) + } + }, + (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), + (None, _) => (None, None), } + }; + let mut report = store.readiness(merge_request::ReadinessCheck { + merge_request_id, + ticket_id, + current_subject_ref, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: String::new(), + worker_id: String::new(), + assignment_id: String::new(), }, - (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), - (None, _) => (None, None), + })?; + if let Some(blocker) = source_blocker { + report + .blockers + .retain(|current| current != "selector_unresolved"); + if !report.blockers.contains(&blocker) { + report.blockers.push(blocker); + } + report.ready = false; } - }; - let mut report = store.readiness(merge_request::ReadinessCheck { - merge_request_id, - ticket_id, - current_subject_ref, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: String::new(), - worker_id: String::new(), - assignment_id: String::new(), - }, - })?; - if let Some(blocker) = source_blocker { - report - .blockers - .retain(|current| current != "selector_unresolved"); - if !report.blockers.contains(&blocker) { - report.blockers.push(blocker); - } - report.ready = false; - } - Ok(Json(server_api::MergeRequestReadinessResponse { - ready: report.ready, - blockers: report.blockers, - subject_ref: report.subject_ref, - review: report.review.map(public_review_event), - })) + Ok(Json(server_api::MergeRequestReadinessResponse { + ready: report.ready, + blockers: report.blockers, + subject_ref: report.subject_ref, + review: report.review.map(public_review_event), + })) + }) + .await } async fn scoped_open_merge_request( @@ -16519,71 +17143,79 @@ async fn scoped_open_merge_request( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let repository = api.require_configured_workspace_repository_by_key(&input.repository_key)?; - let repository_id = repository.id; - require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_from, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &repository_id, - &input.selector_from, - &source_observation.resolved_ref, - )?; - observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_to, - )?; - let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( - merge_request::OpenMergeRequest { - merge_request_id: Uuid::now_v7().to_string(), - ticket_id, - repository_id: repository_id.clone(), - selector_from: input.selector_from, - selector_to: input.selector_to, - summary: input.summary, - auth: merge_request::MergeRequestAuth { - workspace_id: workspace_id.clone(), - repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }, - )?; - Ok(Json(public_merge_request( - &api, - &workspace_id, - merge_request, - )?)) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let repository = + api.require_configured_workspace_repository_by_key(&input.repository_key)?; + let repository_id = repository.id; + require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_from, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &repository_id, + &input.selector_from, + &source_observation.resolved_ref, + )?; + observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_to, + )?; + let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( + merge_request::OpenMergeRequest { + merge_request_id: Uuid::now_v7().to_string(), + ticket_id, + repository_id: repository_id.clone(), + selector_from: input.selector_from, + selector_to: input.selector_to, + summary: input.summary, + auth: merge_request::MergeRequestAuth { + workspace_id: workspace_id.clone(), + repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }, + )?; + Ok(Json(public_merge_request( + &api, + &workspace_id, + merge_request, + )?)) + }) + .await } async fn scoped_merge_request_thread( @@ -16591,19 +17223,24 @@ async fn scoped_merge_request_thread( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult>> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( - &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - Ok(Json( - events - .into_iter() - .map(public_merge_request_thread_event) - .collect(), - )) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( + &workspace_id, + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + Ok(Json( + events + .into_iter() + .map(public_merge_request_thread_event) + .collect(), + )) + }) + .await } async fn scoped_repair_merge_request_selector( @@ -16658,20 +17295,25 @@ async fn scoped_register_reviewer_child_session( AxumPath(workspace_id): AxumPath, Json(input): Json, ) -> ApiResult { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( - merge_request::RegisterReviewerChildSession { - workspace_id, - parent_runtime_id: source.runtime_id, - parent_worker_id: source.worker_id, - child_session_id: input.child_session_id, - reviewer_profile: "builtin:reviewer".into(), - now: Utc::now(), - }, - )?; - Ok(StatusCode::NO_CONTENT) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( + merge_request::RegisterReviewerChildSession { + workspace_id, + parent_runtime_id: source.runtime_id, + parent_worker_id: source.worker_id, + child_session_id: input.child_session_id, + reviewer_profile: "builtin:reviewer".into(), + now: Utc::now(), + }, + )?; + Ok(StatusCode::NO_CONTENT) + }) + .await } async fn scoped_register_merge_request_review_capability( @@ -16680,107 +17322,115 @@ async fn scoped_register_merge_request_review_capability( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let requested_ticket_id = - resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; - if requested_ticket_id != ticket_id { - return Err(Error::TicketAssignmentConflict( - "Reviewer handoff Ticket does not match the addressed Merge Request".into(), - ) - .into()); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &mr.repository_id, - selector, - &source_observation.resolved_ref, - )?; - source_observation.resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err( - Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), - ); - } - }; - let mut ticket_merge_request_subjects = Vec::new(); - for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { - let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged { - merged_result_source_ref(&linked)? - } else { - let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { - Error::InvalidInput(format!( - "Merge Request `{}` selector_from requires repair", - linked.merge_request_id - )) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let requested_ticket_id = + resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; + if requested_ticket_id != ticket_id { + return Err(Error::TicketAssignmentConflict( + "Reviewer handoff Ticket does not match the addressed Merge Request".into(), + ) + .into()); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput("selector_from requires repair".into()) + })?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &mr.repository_id, + selector, + &source_observation.resolved_ref, + )?; + source_observation.resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err(Error::InvalidInput( + "closed Merge Request is not reviewable".into(), + ) + .into()); + } + }; + let mut ticket_merge_request_subjects = Vec::new(); + for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { + let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged + { + merged_result_source_ref(&linked)? + } else { + let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput(format!( + "Merge Request `{}` selector_from requires repair", + linked.merge_request_id + )) + })?; + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &linked.repository_id, + linked_selector, + )? + .resolved_ref + }; + ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { + merge_request_id: linked.merge_request_id, + subject_ref: linked_subject_ref, + }); + } + let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; + store.request_review(merge_request::RequestMergeRequestReview { + merge_request_id, + ticket_id, + ticket_content_digest, + ticket_merge_request_subjects, + subject_ref, + child_session_id: input.child_session_id, + capability_token: input.capability_token, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), })?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &linked.repository_id, - linked_selector, - )? - .resolved_ref - }; - ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { - merge_request_id: linked.merge_request_id, - subject_ref: linked_subject_ref, - }); - } - let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; - store.request_review(merge_request::RequestMergeRequestReview { - merge_request_id, - ticket_id, - ticket_content_digest, - ticket_merge_request_subjects, - subject_ref, - child_session_id: input.child_session_id, - capability_token: input.capability_token, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - })?; - Ok(StatusCode::NO_CONTENT) + Ok(StatusCode::NO_CONTENT) + }) + .await } async fn scoped_submit_merge_request_review( @@ -16788,67 +17438,74 @@ async fn scoped_submit_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let review_authorization = - store.authorize_review_submission(&merge_request_id, &input.capability_token)?; - if review_authorization.workspace_id != workspace_id - || review_authorization.merge_request_id != merge_request_id - { - return Err( - Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( - "review grant invalid".into(), - )) - .into(), - ); - } - let ticket_id = review_authorization.ticket_id; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - let current_subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err( - Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), - ); - } - }; - let decision = internal_review_decision(input.decision); - let findings = input - .findings - .into_iter() - .map(internal_review_finding) - .collect(); - let event = store.submit_review(merge_request::SubmitMergeRequestReview { - merge_request_id, - ticket_id, - current_subject_ref, - capability_token: input.capability_token, - decision, - body: input.body, - findings, - now: Utc::now(), - })?; - Ok(Json(public_review_event(event))) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let review_authorization = + store.authorize_review_submission(&merge_request_id, &input.capability_token)?; + if review_authorization.workspace_id != workspace_id + || review_authorization.merge_request_id != merge_request_id + { + return Err( + Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( + "review grant invalid".into(), + )) + .into(), + ); + } + let ticket_id = review_authorization.ticket_id; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + let current_subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput("selector_from requires repair".into()) + })?; + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err(Error::InvalidInput( + "closed Merge Request is not reviewable".into(), + ) + .into()); + } + }; + let decision = internal_review_decision(input.decision); + let findings = input + .findings + .into_iter() + .map(internal_review_finding) + .collect(); + let event = store.submit_review(merge_request::SubmitMergeRequestReview { + merge_request_id, + ticket_id, + current_subject_ref, + capability_token: input.capability_token, + decision, + body: input.body, + findings, + now: Utc::now(), + })?; + Ok(Json(public_review_event(event))) + }) + .await } async fn scoped_revoke_merge_request_review( @@ -16857,44 +17514,51 @@ async fn scoped_revoke_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - if !input.explicit_confirmation { - return Err(Error::BrowserReopenConfirmationRequired.into()); - } - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let event = store.revoke_review(merge_request::RevokeMergeRequestReview { - merge_request_id, - ticket_id, - review_event_id: input.review_event_id, - reason: input.reason, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - })?; - Ok(Json(public_review_revoked_event(event))) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + if !input.explicit_confirmation { + return Err(Error::BrowserReopenConfirmationRequired.into()); + } + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let event = store.revoke_review(merge_request::RevokeMergeRequestReview { + merge_request_id, + ticket_id, + review_event_id: input.review_event_id, + reason: input.reason, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + })?; + Ok(Json(public_review_revoked_event(event))) + }) + .await } async fn scoped_complete_merge_request( @@ -16903,94 +17567,101 @@ async fn scoped_complete_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - require_online_workspace_orchestrator_source(&api, &source)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { - let replay = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: existing.approved_source_ref.clone(), - target_ref_before: input.target_ref_before, - target_ref_after: input.target_ref_after, - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: String::new(), - }, - now: Utc::now(), - }; - return store - .complete(replay) - .map(public_merge_event) - .map(Json) - .map_err(Into::into); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - let current_source_ref = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref; - let observed_target = observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - )?; - require_completed_target_observation( - &observed_target.resolved_ref, - &input.target_ref_before, - &input.target_ref_after, - )?; - let completion = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: current_source_ref, - target_ref_before: input.target_ref_before.clone(), - target_ref_after: input.target_ref_after.clone(), - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }; - store.validate_completion(&completion)?; - store - .complete(completion) - .map(public_merge_event) - .map(Json) - .map_err(Into::into) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + require_online_workspace_orchestrator_source(&api, &source)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { + let replay = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: existing.approved_source_ref.clone(), + target_ref_before: input.target_ref_before, + target_ref_after: input.target_ref_after, + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: String::new(), + }, + now: Utc::now(), + }; + return store + .complete(replay) + .map(public_merge_event) + .map(Json) + .map_err(Into::into); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + let selector = mr + .selector_from + .as_deref() + .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; + let current_source_ref = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref; + let observed_target = observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + )?; + require_completed_target_observation( + &observed_target.resolved_ref, + &input.target_ref_before, + &input.target_ref_after, + )?; + let completion = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: current_source_ref, + target_ref_before: input.target_ref_before.clone(), + target_ref_after: input.target_ref_after.clone(), + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }; + store.validate_completion(&completion)?; + store + .complete(completion) + .map(public_merge_event) + .map(Json) + .map_err(Into::into) + }) + .await } // Header-only entry point is retained solely for explicit legacy internal tests. @@ -17013,47 +17684,56 @@ async fn scoped_ticket_assignment_tracking_update_with_context( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(update): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let source = authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { - Error::WorkspacePermissionDenied( - "Tracking is self-reported by a verified Runtime Worker".into(), - ) - })?; - // Tracking authority is retained responsibility plus the registered singleton, - // NOT Ticket openness, active work, Worker state or a model-supplied role. - if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { - return Err(Error::WorkspacePermissionDenied( - "Tracking updates require the registered current Workspace Orchestrator".into(), - ) - .into()); - } - let authentication_binding_id = context - .runtime_source - .as_ref() - .map(|source| source.authentication_binding_id.clone()) - .ok_or_else(|| { - Error::WorkspacePermissionDenied( + let context = context.clone(); + api.store + .db_executor() + .run(move || { + let context = &context; + + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let source = + authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Tracking is self-reported by a verified Runtime Worker".into(), + ) + })?; + // Tracking authority is retained responsibility plus the registered singleton, + // NOT Ticket openness, active work, Worker state or a model-supplied role. + if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { + return Err(Error::WorkspacePermissionDenied( + "Tracking updates require the registered current Workspace Orchestrator".into(), + ) + .into()); + } + let authentication_binding_id = context + .runtime_source + .as_ref() + .map(|source| source.authentication_binding_id.clone()) + .ok_or_else(|| { + Error::WorkspacePermissionDenied( "Tracking requires the Runtime binding actually used to authenticate the request" .into(), ) - })?; - let source_context = ticket::TicketAssignmentTrackingSourceContext { - actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - agent_key: "workspace-orchestrator".into(), - authentication_binding_id, - }; - // The Ticket-owned mutation checks current assignment/CAS and exact replay in - // one transaction. A historical receipt never restores operational grants. - let value = api.store.update_ticket_assignment_tracking( - &workspace_id, - &ticket_id, - &update, - &source_context, - )?; - Ok(Json(value)) + })?; + let source_context = ticket::TicketAssignmentTrackingSourceContext { + actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + agent_key: "workspace-orchestrator".into(), + authentication_binding_id, + }; + // The Ticket-owned mutation checks current assignment/CAS and exact replay in + // one transaction. A historical receipt never restores operational grants. + let value = api.store.update_ticket_assignment_tracking( + &workspace_id, + &ticket_id, + &update, + &source_context, + )?; + Ok(Json(value)) + }) + .await } async fn scoped_ticket_assignment_tracking_history( @@ -17061,25 +17741,31 @@ async fn scoped_ticket_assignment_tracking_history( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let limit = query.limit.unwrap_or(50); - if !(1..=100).contains(&limit) { - return Err( - Error::InvalidInput("tracking history limit must be between 1 and 100".into()).into(), - ); - } - let page = api.store.page_ticket_assignment_tracking_history( - &workspace_id, - &ticket_id, - &query.assignment_id, - query.after_event_id.as_deref(), - limit, - )?; - Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { - items: page.items, - next_event_id: page.next_event_id, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let limit = query.limit.unwrap_or(50); + if !(1..=100).contains(&limit) { + return Err(Error::InvalidInput( + "tracking history limit must be between 1 and 100".into(), + ) + .into()); + } + let page = api.store.page_ticket_assignment_tracking_history( + &workspace_id, + &ticket_id, + &query.assignment_id, + query.after_event_id.as_deref(), + limit, + )?; + Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { + items: page.items, + next_event_id: page.next_event_id, + })) + }) + .await } async fn scoped_ticket_case_update_with_context( @@ -17148,6 +17834,10 @@ async fn apply_ticket_case_update( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(mut request): Json, ) -> ApiResult> { + let actor = actor.cloned(); + api.store.db_executor().run(move || { + let actor = actor.as_ref(); + validate_workspace_scope(&api, &workspace_id)?; let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; let mut backend = browser_ticket_backend(&api)?; @@ -17204,6 +17894,8 @@ async fn apply_ticket_case_update( .update_case(&ticket_id, request) .map_err(Error::from)?; Ok(Json(ticket)) + + }).await } fn reject_non_browser_reopen_auth(headers: &HeaderMap) -> Result<()> { @@ -17771,44 +18463,93 @@ fn runtime_local_owner_worker_id<'a>( (caller.runtime_id == target_runtime_id).then_some(caller.worker_id.as_str()) } +// Locked async workflows stay on Tokio. Each accepted synchronous mutation +// retains this same owned guard through execution and discarded-result cleanup. +async fn run_worker_workdir_db_stage_locked( + api: &WorkspaceApi, + session_guard: &Arc>, + operation: impl FnOnce(WorkspaceApi) -> std::result::Result + Send + 'static, +) -> std::result::Result +where + T: Send + 'static, + E: From + Send + 'static, +{ + let stage_api = api.clone(); + let stage_guard = session_guard.clone(); + let (outcome, _session_guard) = api + .store + .db_executor() + .run(move || { + let outcome = operation(stage_api); + // Keep the guard with the outcome, including errors. If the waiter was + // cancelled, the job drops the outcome before releasing this guard. + Ok::<_, E>((outcome, stage_guard)) + }) + .await?; + outcome +} + async fn open_current_worker_workdir_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, ) -> Result { - let workdir = api + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_link = link.clone(); + let (workdir, effective_capabilities) = api .store - .list_workdir_registry(&api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == link.workdir_id) - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!( - "attached Workdir {} is not registered in this Workspace", - link.workdir_id - ), - })?; - if matches!( - workdir.source, - WorkdirRegistrySource::WorkspaceConfig { .. } - ) { - return Err(Error::WorkspacePermissionDenied( - "Logical config has no filesystem session".into(), - )); - } - let effective_capabilities = - effective_worker_workdir_capabilities(api, worker, &workdir, link.capabilities)?; + .db_executor() + .run(move || { + let workdir = stage_api + .store + .list_workdir_registry(&stage_api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == stage_link.workdir_id) + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: stage_worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!( + "attached Workdir {} is not registered in this Workspace", + stage_link.workdir_id + ), + })?; + if matches!( + workdir.source, + WorkdirRegistrySource::WorkspaceConfig { .. } + ) { + return Err(Error::WorkspacePermissionDenied( + "Logical config has no filesystem session".into(), + )); + } + let capabilities = effective_worker_workdir_capabilities( + &stage_api, + &stage_worker, + &workdir, + stage_link.capabilities, + )?; + Ok((workdir, capabilities)) + }) + .await?; if let WorkdirRegistrySource::ExternalGrant { grant_id } = &workdir.source { close_current_worker_attachment_session_locked(api, worker).await?; + let stage_api = api.clone(); + let stage_grant_id = grant_id.clone(); let mut grant = api .store - .get_external_workdir_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "external_workdir_grant_not_found".to_string(), - message: "External Workdir grant is unavailable".to_string(), - })?; + .db_executor() + .run(move || { + stage_api + .store + .get_external_workdir_grant(&stage_api.config.workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "external_workdir_grant_not_found".to_string(), + message: "External Workdir grant is unavailable".to_string(), + }) + }) + .await?; let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(api, grant_id) @@ -17816,9 +18557,13 @@ async fn open_current_worker_workdir_session_locked( .map_err(|error| error.error)?; } if matches!(grant.status.as_str(), "revoked" | "expired") { - cleanup_external_workdir_attachments_with_lock(api, &grant.workdir_id, Some(worker)) - .await - .map_err(|error| error.error)?; + cleanup_external_workdir_attachments_with_lock( + api, + &grant.workdir_id, + Some((worker, session_guard)), + ) + .await + .map_err(|error| error.error)?; cancel_external_workdir_expiry(api, grant_id); } if grant.status != "online" { @@ -17885,31 +18630,63 @@ async fn open_current_worker_workdir_session_locked( }); } }; + let stage_api = api.clone(); + let stage_repository_id = repository_id.to_string(); let repository = api - .require_configured_workspace_repository(repository_id) - .map_err(|error| error.error)?; + .store + .db_executor() + .run(move || { + stage_api + .require_configured_workspace_repository(&stage_repository_id) + .map_err(|error| error.error) + }) + .await?; let repository_requires_access = repository.source.kind == server_api::RepositorySourceKind::Ssh; if repository_requires_access { close_current_worker_attachment_session_locked(api, worker).await?; - let access = repository_access_request_for_workdir( - api, - runtime_id, - &workdir.workdir_id, - &format!( - "workdir-session:{}:{}:{}", - worker.runtime_id, worker.worker_id, workdir.workdir_id - ), - ) - .map_err(|error| error.error)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: runtime_id.to_string(), - code: "working_directory_remote_repository_access_required".to_string(), - message: "current Repository SSH access authority is unavailable".to_string(), - })?; - api.runtime - .authorize_working_directory_repository_access(runtime_id, access) - .map_err(RuntimeRegistryError::into_error)?; + let stage_api = api.clone(); + let stage_runtime_id = runtime_id.to_string(); + let stage_workdir_id = workdir.workdir_id.clone(); + let operation_id = format!( + "workdir-session:{}:{}:{}", + worker.runtime_id, worker.worker_id, workdir.workdir_id + ); + let stage_guard = session_guard.clone(); + let access = api + .store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + repository_access_request_for_workdir( + &stage_api, + &stage_runtime_id, + &stage_workdir_id, + &operation_id, + ) + .map_err(|error| error.error)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: stage_runtime_id, + code: "working_directory_remote_repository_access_required".to_string(), + message: "current Repository SSH access authority is unavailable".to_string(), + }) + }) + .await?; + let stage_api = api.clone(); + let stage_runtime_id = runtime_id.to_string(); + // The accepted Runtime authorization must retain the actual session lock, + // not just the async waiter that submitted it. + let stage_guard = session_guard.clone(); + api.host_executors + .host()? + .run(move || { + let _session_guard = stage_guard; + stage_api + .runtime + .authorize_working_directory_repository_access(&stage_runtime_id, access) + .map_err(RuntimeRegistryError::into_error) + }) + .await?; } else { // The registry is an operation-session cache, not routing authority. // Close any prior alias session before opening the explicitly selected @@ -18038,10 +18815,10 @@ fn restore_logical_workdir_attachments( .map_err(ApiError::from) } -fn sync_runtime_worker_workdir_attachments( +fn prepare_runtime_worker_workdir_attachments( api: &WorkspaceApi, worker: &RuntimeWorkerRef, -) -> ApiResult<()> { +) -> ApiResult> { let attachments = api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? @@ -18084,6 +18861,14 @@ fn sync_runtime_worker_workdir_attachments( }) }) .collect::>>()?; + Ok(attachments) +} + +fn replace_runtime_worker_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + attachments: Vec, +) -> ApiResult<()> { let result = api .runtime .replace_worker_workdir_attachments(worker, attachments) @@ -18103,12 +18888,68 @@ fn sync_runtime_worker_workdir_attachments( Ok(()) } +#[cfg(test)] +fn sync_runtime_worker_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ApiResult<()> { + let attachments = prepare_runtime_worker_workdir_attachments(api, worker)?; + replace_runtime_worker_workdir_attachments(api, worker, attachments) +} + +async fn sync_runtime_worker_workdir_attachments_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + session_guard: &Arc>, +) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.clone(); + let attachments = api + .store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + prepare_runtime_worker_workdir_attachments(&stage_api, &stage_worker) + }) + .await?; + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.clone(); + api.host_executors + .host()? + .run(move || { + let _session_guard = stage_guard; + replace_runtime_worker_workdir_attachments(&stage_api, &stage_worker, attachments) + }) + .await +} + async fn refresh_current_worker_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, + session_guard: &Arc>, ) -> Result<()> { close_current_worker_attachment_session_locked(api, worker).await?; + let stage_api = api.clone(); + let stage_worker = worker.clone(); let remaining = api + .store + .db_executor() + .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) + .await?; + if remaining.len() == 1 { + open_current_worker_workdir_session_locked(api, worker, &remaining[0], session_guard) + .await?; + } + Ok(()) +} + +fn active_filesystem_workdir_links( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Result> { + Ok(api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() @@ -18122,11 +18963,7 @@ async fn refresh_current_worker_session_locked( !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) }) }) - .collect::>(); - if remaining.len() == 1 { - open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await?; - } - Ok(()) + .collect()) } #[derive(Debug, Deserialize)] @@ -18141,35 +18978,107 @@ async fn scoped_submit_backend_job_result( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; - let binding = api - .store - .worker_registry_projection(&api.config.workspace_id, &worker)? - .and_then(|projection| projection.job) - .ok_or_else(|| { - Error::InvalidInput( - "current Worker is not bound to an active Backend Job attempt".to_string(), - ) - })?; - let attempt = api - .store - .get_backend_job_attempt( + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let binding = api + .store + .worker_registry_projection(&api.config.workspace_id, &worker)? + .and_then(|projection| projection.job) + .ok_or_else(|| { + Error::InvalidInput( + "current Worker is not bound to an active Backend Job attempt".to_string(), + ) + })?; + let attempt = api + .store + .get_backend_job_attempt( + &api.config.workspace_id, + &binding.job_id, + &binding.attempt_id, + )? + .ok_or_else(|| { + Error::Store("Backend Job attempt binding disappeared".to_string()) + })?; + let acceptance = api.accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + input_digest: attempt.input_digest, + result: request.result, + }, + )?; + Ok(Json(acceptance)) + }) + .await +} + +async fn detach_worker_workdir_stage_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + workdir_id: &str, + session_guard: &Arc>, +) -> Result> { + let worker = worker.clone(); + let workdir_id = workdir_id.to_string(); + run_worker_workdir_db_stage_locked(api, session_guard, move |api| { + api.store.detach_worker_workdir( &api.config.workspace_id, - &binding.job_id, - &binding.attempt_id, - )? - .ok_or_else(|| Error::Store("Backend Job attempt binding disappeared".to_string()))?; - let acceptance = api.accept_backend_job_result( - &worker, - &BackendJobResultSubmission { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - input_digest: attempt.input_digest, - result: request.result, - }, - )?; - Ok(Json(acceptance)) + &worker, + Some(&workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) + }) + .await +} + +async fn import_current_worker_workdir_observation( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + session_guard: Option<&Arc>>, +) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let has_history = api + .store + .db_executor() + .run(move || { + stage_api + .store + .worker_workdir_link_history_exists(&stage_api.config.workspace_id, &stage_worker) + }) + .await?; + if !has_history { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.cloned(); + let observed_worker = api + .host_executors + .host()? + .run(move || { + let _session_guard = stage_guard; + stage_api + .runtime + .worker(&stage_worker) + .map_err(RuntimeRegistryError::into_error) + }) + .await?; + if !observed_worker.workdir_attachments.is_empty() { + let stage_api = api.clone(); + let stage_guard = session_guard.cloned(); + api.store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + sync_worker_observation(&stage_api, &observed_worker) + }) + .await?; + } + } + Ok(()) } async fn scoped_attach_current_worker_workdir( @@ -18178,115 +19087,141 @@ async fn scoped_attach_current_worker_workdir( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host()? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &path.workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &path.workspace_id, + &headers, + )?) + }) + .await?; let alias = workdir::WorkdirAttachmentAlias::new(request.alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; let workdir_id = request.working_directory_id.trim(); if workdir_id.is_empty() || workdir_id.chars().any(char::is_control) { return Err(Error::InvalidRecordId(request.working_directory_id).into()); } - let workdir = api + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_workdir_id = workdir_id.to_string(); + let capabilities = api .store - .list_workdir_registry(&api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == workdir_id) - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!("unknown Workdir `{workdir_id}`"), - })?; - // A failed removal must not prevent its owner from attaching a still usable - // repository checkout to resolve current blockers. Runtime binding validates - // the actual checkout; durable pending-removal reservations still fence this - // mutation. Other sources and missing/corrupted materializations stay denied. - let repository_retry = workdir.materialization_status == "pending" - && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); - if workdir.materialization_status != "present" && !repository_retry { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir `{workdir_id}` is not available for attachment ({})", - workdir.materialization_status - )) - .into()); - } - match &workdir.source { - WorkdirRegistrySource::Repository { .. } => {} - WorkdirRegistrySource::WorkspaceConfig { .. } => { - return Err(Error::WorkspacePermissionDenied( - "Use the grant-bound Workspace config attachment operation".into(), - ) - .into()); - } - WorkdirRegistrySource::ExternalGrant { grant_id } => { - let grant = api + .db_executor() + .run(move || -> ApiResult<_> { + let api = stage_api; + let worker = stage_worker; + let workdir_id = stage_workdir_id.as_str(); + let workdir = api .store - .get_external_workdir_grant(&api.config.workspace_id, grant_id)? + .list_workdir_registry(&api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == workdir_id) .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir grant is unavailable".to_string(), + runtime_id: worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!("unknown Workdir `{workdir_id}`"), })?; - let grant_expired = external_workdir_grant_is_expired(&grant)?; - let online = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connection(grant_id) - .is_some_and(|connection| { - connection.generation == grant.generation - && connection.provider_instance_id == grant.provider_instance_id - && connection.workdir_id == workdir.workdir_id - && connection - .expires_at - .is_none_or(|expires_at| expires_at > Utc::now()) - }); - if grant.status != "online" || grant_expired || !online { - return Err(Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir provider is offline or expired".to_string(), - } + // A failed removal must not prevent its owner from attaching a still usable + // repository checkout to resolve current blockers. Runtime binding validates + // the actual checkout; durable pending-removal reservations still fence this + // mutation. Other sources and missing/corrupted materializations stay denied. + let repository_retry = workdir.materialization_status == "pending" + && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); + if workdir.materialization_status != "present" && !repository_retry { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir `{workdir_id}` is not available for attachment ({})", + workdir.materialization_status + )) .into()); } - } - } - let capabilities = ticket_target_capabilities_for_workdir(&api, &worker, &workdir)?; + match &workdir.source { + WorkdirRegistrySource::Repository { .. } => {} + WorkdirRegistrySource::WorkspaceConfig { .. } => { + return Err(Error::WorkspacePermissionDenied( + "Use the grant-bound Workspace config attachment operation".into(), + ) + .into()); + } + WorkdirRegistrySource::ExternalGrant { grant_id } => { + let grant = api + .store + .get_external_workdir_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir grant is unavailable".to_string(), + })?; + let grant_expired = external_workdir_grant_is_expired(&grant)?; + let online = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connection(grant_id) + .is_some_and(|connection| { + connection.generation == grant.generation + && connection.provider_instance_id == grant.provider_instance_id + && connection.workdir_id == workdir.workdir_id + && connection + .expires_at + .is_none_or(|expires_at| expires_at > Utc::now()) + }); + if grant.status != "online" || grant_expired || !online { + return Err(Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir provider is offline or expired".to_string(), + } + .into()); + } + } + } + Ok(ticket_target_capabilities_for_workdir( + &api, &worker, &workdir, + )?) + }) + .await?; let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; - if !api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, &worker)? - { - let observed_worker = api - .runtime - .worker(&worker) - .map_err(|error| error.into_error())?; - if !observed_worker.workdir_attachments.is_empty() { - sync_worker_observation(&api, &observed_worker)?; - } - } - if let Some(existing) = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == alias.as_str()) - { - if existing.workdir_id != workdir_id { - return Err(Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} attachment alias `{alias}` is already bound to Workdir {}", - worker.runtime_id, worker.worker_id, existing.workdir_id - )) - .into()); - } - let effective_capabilities = existing.capabilities.intersection(capabilities); - if effective_capabilities != existing.capabilities { - let mut attenuated = existing.clone(); - attenuated.capabilities = effective_capabilities; - api.store.attach_worker_workdir(&attenuated)?; - } - sync_runtime_worker_workdir_attachments(&api, &worker)?; + let _session_guard = Arc::new(session_lock.lock_owned().await); + import_current_worker_workdir_observation(&api, &worker, Some(&_session_guard)).await?; + let stage_worker = worker.clone(); + let stage_alias = alias.to_string(); + let stage_workdir_id = workdir_id.to_string(); + let existing = + run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| -> ApiResult<_> { + let existing = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias); + if let Some(existing) = existing { + if existing.workdir_id != stage_workdir_id { + return Err(Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} attachment alias `{stage_alias}` is already bound to Workdir {}", + stage_worker.runtime_id, stage_worker.worker_id, existing.workdir_id + )).into()); + } + let effective_capabilities = existing.capabilities.intersection(capabilities); + if effective_capabilities != existing.capabilities { + let mut attenuated = existing; + attenuated.capabilities = effective_capabilities; + api.store.attach_worker_workdir(&attenuated)?; + } + Ok(Some(effective_capabilities)) + } else { + Ok(None) + } + }) + .await?; + if let Some(effective_capabilities) = existing { + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; api.worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from)?; return Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { workspace_id: api.config.workspace_id.clone(), @@ -18297,7 +19232,7 @@ async fn scoped_attach_current_worker_workdir( })); } close_current_worker_attachment_session_locked(&api, &worker).await?; - let link = api.store.attach_worker_workdir(&WorkerWorkdirLinkRecord { + let new_link = WorkerWorkdirLinkRecord { connection_id: String::new(), workspace_id: api.config.workspace_id.clone(), worker: worker.clone(), @@ -18306,56 +19241,46 @@ async fn scoped_attach_current_worker_workdir( capabilities, linked_at: Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), unlinked_at: None, - })?; - if let Err(error) = sync_runtime_worker_workdir_attachments(&api, &worker) { - let _ = api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ); + }; + let link = run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| { + api.store.attach_worker_workdir(&new_link) + }) + .await?; + if let Err(error) = + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await + { + let _ = + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; return Err(error); } + let stage_api = api.clone(); + let stage_worker = worker.clone(); let active_count = api .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) - .filter(|link| { - api.store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) - .ok() - .flatten() - .is_none_or(|record| { - !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) - }) + .db_executor() + .run(move || { + active_filesystem_workdir_links(&stage_api, &stage_worker).map(|links| links.len()) }) - .count(); + .await?; if active_count == 1 - && let Err(error) = open_current_worker_workdir_session_locked(&api, &worker, &link).await + && let Err(error) = + open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await { - let _ = api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ); - let _ = sync_runtime_worker_workdir_attachments(&api, &worker); + let _ = + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; + let _ = + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await; return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from) { - api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - )?; - sync_runtime_worker_workdir_attachments(&api, &worker)?; - refresh_current_worker_session_locked(&api, &worker).await?; + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await?; + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; + refresh_current_worker_session_locked(&api, &worker, &_session_guard).await?; return Err(error); } Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -18485,7 +19410,12 @@ async fn scoped_detach_current_worker_workdir( validate_workspace_scope(&api, &workspace_id)?; let alias = workdir::WorkdirAttachmentAlias::new(alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; - let worker = current_worker_identity(&api, &workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host()? + .run(move || current_worker_identity(&stage_api, &workspace_id, &headers)) + .await?; detach_current_worker_workdir( &api, &worker, @@ -18496,6 +19426,18 @@ async fn scoped_detach_current_worker_workdir( .map(Json) } +async fn restore_worker_workdir_connection_stage_locked( + api: &WorkspaceApi, + link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, +) -> Result<()> { + let link = link.clone(); + run_worker_workdir_db_stage_locked(api, session_guard, move |api| { + api.store.restore_worker_workdir_connection(&link) + }) + .await +} + async fn detach_current_worker_workdir( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -18503,70 +19445,84 @@ async fn detach_current_worker_workdir( expected_connection_id: Option<&str>, ) -> ApiResult { let session_lock = current_worker_session_lock(api, worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_alias = alias.to_string(); + let expected_connection_id = expected_connection_id.map(str::to_string); let link = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == alias) - .ok_or_else(|| { - Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} has no Workdir attachment `{alias}`", - worker.runtime_id, worker.worker_id - )) - })?; - if expected_connection_id.is_some_and(|expected| expected != link.connection_id) { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir attachment `{alias}` connection changed during detach" - )) - .into()); - } + .db_executor() + .run(move || -> ApiResult<_> { + let link = stage_api + .store + .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias) + .ok_or_else(|| { + Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} has no Workdir attachment `{stage_alias}`", + stage_worker.runtime_id, stage_worker.worker_id + )) + })?; + if expected_connection_id + .as_ref() + .is_some_and(|expected| *expected != link.connection_id) + { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir attachment `{stage_alias}` connection changed during detach" + )) + .into()); + } + Ok(link) + }) + .await?; close_current_worker_attachment_session_locked(api, worker).await?; close_current_worker_command_sessions_for_alias_locked(api, worker, alias).await?; // Even unconditional HTTP requests condition the mutation on the lifetime just read // under the session lock, so a concurrent ledger replacement cannot be removed. - api.store.detach_worker_workdir_connection( - &api.config.workspace_id, - worker, - alias, - &link.connection_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - )?; - if let Err(error) = sync_runtime_worker_workdir_attachments(api, worker) { - api.store.restore_worker_workdir_connection(&link)?; + let stage_link = link.clone(); + run_worker_workdir_db_stage_locked(api, &_session_guard, move |api| { + api.store.detach_worker_workdir_connection( + &api.config.workspace_id, + &stage_link.worker, + &stage_link.alias, + &stage_link.connection_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) + }) + .await?; + if let Err(error) = + sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await + { + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; return Err(error); } + let stage_api = api.clone(); + let stage_worker = worker.clone(); let remaining = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) - .filter(|link| { - api.store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) - .ok() - .flatten() - .is_none_or(|record| { - !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) - }) - }) - .collect::>(); + .db_executor() + .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) + .await?; if remaining.len() == 1 && let Err(error) = - open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await + open_current_worker_workdir_session_locked(api, worker, &remaining[0], &_session_guard) + .await { - api.store.restore_worker_workdir_connection(&link)?; - let _ = sync_runtime_worker_workdir_attachments(api, worker); + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; + let _ = sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await; return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from) { - api.store.restore_worker_workdir_connection(&link)?; - sync_runtime_worker_workdir_attachments(api, worker)?; - refresh_current_worker_session_locked(api, worker).await?; + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; + sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await?; + refresh_current_worker_session_locked(api, worker, &_session_guard).await?; return Err(error); } Ok(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -18578,41 +19534,36 @@ async fn detach_current_worker_workdir( }) } -fn validated_current_worker_attachment( +async fn validated_current_worker_attachment_async( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, + session_guard: Option<&Arc>>, ) -> ApiResult { - let find = || -> Result> { - Ok(api + for imported in [false, true] { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_alias = target_workdir.to_string(); + let link = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == target_workdir)) - }; - if let Some(link) = find()? { - return Ok(link); - } - - // A Runtime can start the Worker immediately after reserving its local - // bindings, before the outer spawn handler projects them into the Backend. - // Import that exact alias set once; never substitute another attachment. - if !api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, worker)? - { - let observed_worker = api - .runtime - .worker(worker) - .map_err(|error| error.into_error())?; - if !observed_worker.workdir_attachments.is_empty() { - sync_worker_observation(api, &observed_worker)?; - if let Some(link) = find()? { - return Ok(link); - } + .db_executor() + .run(move || -> Result<_> { + Ok(stage_api + .store + .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias)) + }) + .await?; + if let Some(link) = link { + return Ok(link); + } + if !imported { + // A Runtime may start the Worker before the outer spawn handler + // projects its reserved aliases. Import that exact set only once. + import_current_worker_workdir_observation(api, worker, session_guard).await?; } } - Err(Error::WorkdirAttachmentConflict(format!( "Worker {}:{} has no Workdir attachment alias `{target_workdir}`", worker.runtime_id, worker.worker_id @@ -18721,19 +19672,24 @@ async fn scoped_execute_current_worker_workdir_operation( context.stage = "workspace_scope"; validate_workspace_scope(&api, &path.workspace_id)?; context.stage = "worker_identity"; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = api.host_executors.host()?.run(move || { + current_worker_identity(&stage_api, &path.workspace_id, &headers) + }).await?; context.runtime_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.runtime_id)); context.worker_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.worker_id)); let target_workdir = request.target_workdir; let result = match request.operation { WorkdirSessionOperation::CommandStart(command) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; + let link = validated_current_worker_attachment_async( + &api, &worker, &target_workdir, Some(&_session_guard), + ).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; context.stage = "provider_dispatch"; let provider_handle = source .start_command(command) @@ -18765,7 +19721,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandStatus(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; session .command_status(provider_handle) .await @@ -18774,7 +19730,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandOutput(mut output) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context).await?; output.handle = provider_handle; session .command_output(output) @@ -18784,7 +19740,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandCancel(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; session .cancel_command(provider_handle) .await @@ -18805,12 +19761,14 @@ async fn scoped_execute_current_worker_workdir_operation( | WorkdirSessionOperation::Glob(_) | WorkdirSessionOperation::Grep(_)) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; + let link = validated_current_worker_attachment_async( + &api, &worker, &target_workdir, Some(&_session_guard), + ).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; context.stage = "provider_dispatch"; execute_workdir_session_operation(&source, operation) .await @@ -18825,7 +19783,7 @@ async fn scoped_execute_current_worker_workdir_operation( }) } -fn current_worker_command_session( +async fn current_worker_command_session( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, @@ -18833,7 +19791,7 @@ fn current_worker_command_session( context: &mut server_api::WorkdirOperationLogContext, ) -> std::result::Result<(WorkdirSessionHandle, CommandHandle), InternalWorkdirOperationError> { context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(api, worker, target_workdir)?; + let link = validated_current_worker_attachment_async(api, worker, target_workdir, None).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "command_lookup"; let command = api @@ -18939,8 +19897,29 @@ async fn run_orchestrator_turn_end_hook(api: WorkspaceApi) { }; let mut worker_states = HashMap::new(); while let Some(update) = subscription.recv().await { - if !observe_orchestrator_turn_end(&api, &mut worker_states, update) { - return; + let stage_api = api.clone(); + let result = api.host_executors.host().map(|executor| async move { + executor + .run(move || -> Result<_> { + let keep_running = + observe_orchestrator_turn_end(&stage_api, &mut worker_states, update); + Ok((keep_running, worker_states)) + }) + .await + }); + match result { + Ok(operation) => match operation.await { + Ok((true, states)) => worker_states = states, + Ok((false, _)) => return, + Err(error) => { + tracing::warn!(%error, "Orchestrator hook stopped; durable inputs remain recoverable"); + return; + } + }, + Err(error) => { + tracing::warn!(%error, "Orchestrator hook executor unavailable"); + return; + } } } } @@ -19020,6 +19999,27 @@ fn maybe_dispatch_orchestrator_turn_end( } } +fn enqueue_orchestrator_assignment_attention( + api: &WorkspaceApi, +) -> Option> { + let api = api.clone(); + let (done, completion) = tokio::sync::oneshot::channel(); + match api.host_executors.host().and_then(|executor| { + executor.dispatch(move || { + if let Some(orchestrator) = find_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + let _ = done.send(()); + }) + }) { + Ok(()) => Some(completion), + Err(error) => { + tracing::warn!(%error, "Assign notification was not admitted; input remains durable"); + None + } + } +} + fn dispatch_orchestrator_assignment_attention(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { with_orchestrator_attention_reservation(api, worker, true, |state, pending| { check_orchestrator_assignment_attention(api, worker, state, pending); @@ -19360,15 +20360,20 @@ async fn scoped_get_memory_document( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let document = api.authority.memory_document()?; - Ok(Json(MemoryDocumentResponse { - bytes: document.body_md.len(), - body_md: document.body_md, - created_at: document.created_at, - updated_at: document.updated_at, - record_source: document.record_source, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let document = api.authority.memory_document()?; + Ok(Json(MemoryDocumentResponse { + bytes: document.body_md.len(), + body_md: document.body_md, + created_at: document.created_at, + updated_at: document.updated_at, + record_source: document.record_source, + })) + }) + .await } async fn scoped_reset_legacy_memory( @@ -19391,13 +20396,18 @@ async fn scoped_reset_legacy_memory( )) .into()); } - let counts = api.store.reset_legacy_memory(&path.workspace_id)?; - Ok(Json(server_api::LegacyMemoryResetResponse { - workspace_id: path.workspace_id, - documents_deleted: counts.documents, - staging_records_deleted: counts.staging_records, - staging_resolutions_deleted: counts.staging_resolutions, - })) + api.store + .db_executor() + .run(move || { + let counts = api.store.reset_legacy_memory(&path.workspace_id)?; + Ok(Json(server_api::LegacyMemoryResetResponse { + workspace_id: path.workspace_id, + documents_deleted: counts.documents, + staging_records_deleted: counts.staging_records, + staging_resolutions_deleted: counts.staging_resolutions, + })) + }) + .await } async fn scoped_list_memory_staging( @@ -19405,13 +20415,20 @@ async fn scoped_list_memory_staging( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(list_memory_staging_from_authority( - &api.authority, - query.limit.map(usize::try_from).transpose().map_err(|_| { - Error::InvalidInput("Memory staging limit exceeds this host's usize range".to_string()) - })?, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(list_memory_staging_from_authority( + &api.authority, + query.limit.map(usize::try_from).transpose().map_err(|_| { + Error::InvalidInput( + "Memory staging limit exceeds this host's usize range".to_string(), + ) + })?, + )?)) + }) + .await } async fn scoped_memory_backend_operation( @@ -19420,21 +20437,26 @@ async fn scoped_memory_backend_operation( _context: server_api::ServerRequestContext, Json(operation): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - // The legacy memory document edit is a read/replace/write across store - // calls. Keep that resource atomic without serializing unrelated requests. - let _memory_guard = api - .memory_operation_lock - .lock() - .expect("Memory operation lock poisoned"); - let result = execute_memory_backend_operation_with_authority(&api.authority, operation); - let response = match result { - Ok(result) => MemoryBackendHttpResponse::Ok { result }, - Err(error) => MemoryBackendHttpResponse::Error { - message: sanitize_backend_error(&error.to_string()), - }, - }; - Ok(Json(response)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + // The legacy memory document edit is a read/replace/write across store + // calls. Keep that resource atomic without serializing unrelated requests. + let _memory_guard = api + .memory_operation_lock + .lock() + .expect("Memory operation lock poisoned"); + let result = execute_memory_backend_operation_with_authority(&api.authority, operation); + let response = match result { + Ok(result) => MemoryBackendHttpResponse::Ok { result }, + Err(error) => MemoryBackendHttpResponse::Error { + message: sanitize_backend_error(&error.to_string()), + }, + }; + Ok(Json(response)) + }) + .await } const SUBJEKTIV_SINGLETON_PREFIX: &str = crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX; @@ -19442,6 +20464,66 @@ const SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX: &str = crate::hosts::SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX; const SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE: &str = "subjektiv-memory-consolidation"; +// Runtime callbacks may themselves need DB authority. Synchronous Host workflows +// therefore have a separate finite lane: never monopolize the sole DB worker while +// waiting for Runtime I/O. Existing transaction helpers still finish before I/O. +// Teardown waits must not occupy the ordinary Host lane: a Job may need a +// user stop/control operation on that lane before its Run can finish. Cleanup +// therefore has its own finite owner, still independent of the DB callback lane. +#[derive(Clone, Default)] +struct HostExecutors { + host: Arc>>, + cleanup: Arc>>, +} + +impl HostExecutors { + fn shared() -> Self { + static SHARED: std::sync::OnceLock = std::sync::OnceLock::new(); + SHARED.get_or_init(Self::default).clone() + } + + fn lane( + owner: &std::sync::OnceLock>, + ) -> Result { + owner + .get_or_init(|| { + crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(2).unwrap(), + queued: std::num::NonZeroUsize::new(16).unwrap(), + }) + .ok() + }) + .clone() + .ok_or(Error::DbExecutorUnavailable) + } + + fn host(&self) -> Result { + Self::lane(&self.host) + } + fn cleanup(&self) -> Result { + Self::lane(&self.cleanup) + } +} + +// Some existing lifecycle workflows interleave owned async resource locks with +// synchronous DB/Runtime APIs. Poll the complete workflow on the finite Host +// lane using the original runtime (never create a replacement runtime for its +// cached services). Cancellation releases neither locks nor accepted ownership. +// No SQL transaction contains an await/Runtime call, and this lane is separate +// from the DB worker needed by Runtime callbacks. Do not nest this helper. +async fn run_host_workflow( + executor: crate::db_executor::DbExecutor, + future: impl std::future::Future> + Send + 'static, +) -> std::result::Result +where + T: Send + 'static, + E: From + Send + 'static, +{ + let runtime = + tokio::runtime::Handle::try_current().map_err(|_| E::from(Error::DbExecutorUnavailable))?; + executor.run(move || runtime.block_on(future)).await +} + fn open_subjektiv_store(api: &WorkspaceApi) -> ApiResult { crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) .map_err(|error| ApiError::from(Error::Store(error.to_string()))) @@ -19659,18 +20741,23 @@ async fn scoped_create_subjektiv_subject( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = crate::subjektiv::SubjectRole::new(request.role) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .create_subject_with_behavior(role, request.behavior_md) - .map_err(subjektiv_store_error)?; - Ok(( - StatusCode::CREATED, - Json(subjektiv_subject_response(&api, subject)?), - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let role = crate::subjektiv::SubjectRole::new(request.role) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .create_subject_with_behavior(role, request.behavior_md) + .map_err(subjektiv_store_error)?; + Ok(( + StatusCode::CREATED, + Json(subjektiv_subject_response(&api, subject)?), + )) + }) + .await } #[derive(Debug, Serialize, Deserialize)] @@ -19686,76 +20773,86 @@ async fn scoped_list_subjektiv_subjects( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let limit = query.limit.unwrap_or(20); - if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { - return Err(Error::InvalidInput(format!( - "subject list limit must be within 1..={}", - server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT - )) - .into()); - } - let cursor = query - .cursor - .as_deref() - .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) - .transpose()?; - if cursor - .as_ref() - .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) - { - return Err(Error::InvalidInput( - "subjektiv subject cursor does not match Workspace".to_string(), - ) - .into()); - } - let page = open_subjektiv_store(&api)? - .list_subjects_after( - limit, - cursor + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let limit = query.limit.unwrap_or(20); + if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { + return Err(Error::InvalidInput(format!( + "subject list limit must be within 1..={}", + server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT + )) + .into()); + } + let cursor = query + .cursor + .as_deref() + .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) + .transpose()?; + if cursor .as_ref() - .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), - ) - .map_err(subjektiv_store_error)?; - let next_cursor = if page.has_more { - page.items - .last() - .map(|subject| { - encode_subjektiv_cursor( - "subjects", - &SubjektivSubjectCursor { - workspace_id: path.workspace_id.clone(), - created_at: subject.created_at.clone(), - subject_id: subject.id.clone(), - }, + .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) + { + return Err(Error::InvalidInput( + "subjektiv subject cursor does not match Workspace".to_string(), ) - }) - .transpose()? - } else { - None - }; - Ok(Json(server_api::SubjektivSubjectListResponse { - limit, - items: page - .items - .into_iter() - .map(|subject| subjektiv_subject_response(&api, subject)) - .collect::>>()?, - next_cursor, - has_more: page.has_more, - })) + .into()); + } + let page = open_subjektiv_store(&api)? + .list_subjects_after( + limit, + cursor + .as_ref() + .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), + ) + .map_err(subjektiv_store_error)?; + let next_cursor = if page.has_more { + page.items + .last() + .map(|subject| { + encode_subjektiv_cursor( + "subjects", + &SubjektivSubjectCursor { + workspace_id: path.workspace_id.clone(), + created_at: subject.created_at.clone(), + subject_id: subject.id.clone(), + }, + ) + }) + .transpose()? + } else { + None + }; + Ok(Json(server_api::SubjektivSubjectListResponse { + limit, + items: page + .items + .into_iter() + .map(|subject| subjektiv_subject_response(&api, subject)) + .collect::>>()?, + next_cursor, + has_more: page.has_more, + })) + }) + .await } async fn scoped_get_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .subject(&path.subject_id) - .map_err(|error| Error::Store(error.to_string()))? - .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .subject(&path.subject_id) + .map_err(|error| Error::Store(error.to_string()))? + .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } async fn scoped_update_subjektiv_subject_behavior( @@ -19763,17 +20860,22 @@ async fn scoped_update_subjektiv_subject_behavior( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .update_subject_behavior( - &path.subject_id, - &request.expected_behavior_md, - request.behavior_md, - ) - .map_err(subjektiv_store_error)?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .update_subject_behavior( + &path.subject_id, + &request.expected_behavior_md, + request.behavior_md, + ) + .map_err(subjektiv_store_error)?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } fn require_subjektiv_subject( @@ -19794,47 +20896,53 @@ async fn scoped_get_subjektiv_resident_surface( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - let resident = store - .resident_surface(&path.subject_id) - .map_err(subjektiv_store_error)?; - let availability = match resident.availability { - crate::subjektiv::SurfaceAvailability::Ungenerated => { - server_api::SubjektivResidentSurfaceAvailability::Ungenerated - } - crate::subjektiv::SurfaceAvailability::Stale => { - server_api::SubjektivResidentSurfaceAvailability::Stale - } - crate::subjektiv::SurfaceAvailability::Failed => { - server_api::SubjektivResidentSurfaceAvailability::Failed - } - crate::subjektiv::SurfaceAvailability::Ready => { - server_api::SubjektivResidentSurfaceAvailability::Ready - } - }; - let snapshot = resident - .snapshot - .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { - snapshot_id: snapshot.id, - body_md: snapshot.body_md, - memory_refs: snapshot - .memory_refs - .into_iter() - .map(|reference| server_api::SubjektivMemoryChangeRef { - memory_id: reference.memory_id, - change_id: reference.change_id, - }) - .collect(), - built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, - created_at: snapshot.created_at, - }); - Ok(Json(server_api::SubjektivResidentSurfaceResponse { - subject_id: path.subject_id, - availability, - snapshot, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + let resident = store + .resident_surface(&path.subject_id) + .map_err(subjektiv_store_error)?; + let availability = match resident.availability { + crate::subjektiv::SurfaceAvailability::Ungenerated => { + server_api::SubjektivResidentSurfaceAvailability::Ungenerated + } + crate::subjektiv::SurfaceAvailability::Stale => { + server_api::SubjektivResidentSurfaceAvailability::Stale + } + crate::subjektiv::SurfaceAvailability::Failed => { + server_api::SubjektivResidentSurfaceAvailability::Failed + } + crate::subjektiv::SurfaceAvailability::Ready => { + server_api::SubjektivResidentSurfaceAvailability::Ready + } + }; + let snapshot = + resident + .snapshot + .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { + snapshot_id: snapshot.id, + body_md: snapshot.body_md, + memory_refs: snapshot + .memory_refs + .into_iter() + .map(|reference| server_api::SubjektivMemoryChangeRef { + memory_id: reference.memory_id, + change_id: reference.change_id, + }) + .collect(), + built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, + created_at: snapshot.created_at, + }); + Ok(Json(server_api::SubjektivResidentSurfaceResponse { + subject_id: path.subject_id, + availability, + snapshot, + })) + }) + .await } async fn scoped_list_subjektiv_memories( @@ -19842,14 +20950,19 @@ async fn scoped_list_subjektiv_memories( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_query( - &store, - &path.subject_id, - query, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_query( + &store, + &path.subject_id, + query, + )?)) + }) + .await } async fn scoped_get_subjektiv_memory( @@ -19857,21 +20970,26 @@ async fn scoped_get_subjektiv_memory( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_read( - &store, - &path.subject_id, - server_api::SubjektivMemoryReadRequest { - memory_id: path.memory_id, - change_id: query.change_id, - offset: query.offset, - byte_offset: query.byte_offset, - limit: query.limit, - evidence_cursor: query.evidence_cursor, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_read( + &store, + &path.subject_id, + server_api::SubjektivMemoryReadRequest { + memory_id: path.memory_id, + change_id: query.change_id, + offset: query.offset, + byte_offset: query.byte_offset, + limit: query.limit, + evidence_cursor: query.evidence_cursor, + }, + )?)) + }) + .await } async fn scoped_list_subjektiv_memory_changes( @@ -19879,31 +20997,41 @@ async fn scoped_list_subjektiv_memory_changes( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_list_changes( - &store, - &path.subject_id, - server_api::SubjektivMemoryListChangesRequest { - memory_id: path.memory_id, - limit: query.limit, - cursor: query.cursor, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_list_changes( + &store, + &path.subject_id, + server_api::SubjektivMemoryListChangesRequest { + memory_id: path.memory_id, + limit: query.limit, + cursor: query.cursor, + }, + )?)) + }) + .await } async fn scoped_retire_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .retire_subject(&path.subject_id) - .map_err(subjektiv_store_error)?; - // Retirement deliberately does not stop or remove the current Worker. The - // Worker lease and subject lifecycle are independent authorities. - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .retire_subject(&path.subject_id) + .map_err(subjektiv_store_error)?; + // Retirement deliberately does not stop or remove the current Worker. The + // Worker lease and subject lifecycle are independent authorities. + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } async fn scoped_stage_subjektiv_candidate( @@ -19912,58 +21040,63 @@ async fn scoped_stage_subjektiv_candidate( context: server_api::ServerRequestContext, Json(mut request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )?; - let subject_id = attribution.subject_id.clone(); - let session_id = attribution.session_id.as_str(); - for source_ref in &mut request.operation.source_refs { - if source_ref - .session_id - .as_deref() - .is_some_and(|value| value != session_id) - { - return Err(Error::InvalidInput( - "candidate source_ref belongs to a different Session".to_string(), - ) - .into()); - } - source_ref.session_id = Some(session_id.to_string()); - } - if matches!( - request.operation.candidate.kind, - memory::extract::CandidateKind::Preference - ) && request.operation.evidence.iter().any(|evidence| { - !matches!( - evidence.origin.as_ref().map(|origin| &origin.kind), - Some(memory::schema::EvidenceOriginKind::HumanInput) - ) - }) { - return Err(Error::InvalidInput( - "preference candidates require exclusively HumanInput evidence".to_string(), - ) - .into()); - } - let candidate_id = Uuid::now_v7().to_string(); - let record = memory::extract::StagingRecord::from_candidate( - candidate_id, - request.operation.extract_run_id, - request.operation.source, - request.operation.candidate, - request.operation.evidence, - request.operation.source_refs, - ); - let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); - let (staged, _) = open_subjektiv_store(&api)? - .stage_candidate_with_attribution(record, attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - Ok(Json(server_api::SubjektivStageCandidateResponse { - staging_id: staged.id, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )?; + let subject_id = attribution.subject_id.clone(); + let session_id = attribution.session_id.as_str(); + for source_ref in &mut request.operation.source_refs { + if source_ref + .session_id + .as_deref() + .is_some_and(|value| value != session_id) + { + return Err(Error::InvalidInput( + "candidate source_ref belongs to a different Session".to_string(), + ) + .into()); + } + source_ref.session_id = Some(session_id.to_string()); + } + if matches!( + request.operation.candidate.kind, + memory::extract::CandidateKind::Preference + ) && request.operation.evidence.iter().any(|evidence| { + !matches!( + evidence.origin.as_ref().map(|origin| &origin.kind), + Some(memory::schema::EvidenceOriginKind::HumanInput) + ) + }) { + return Err(Error::InvalidInput( + "preference candidates require exclusively HumanInput evidence".to_string(), + ) + .into()); + } + let candidate_id = Uuid::now_v7().to_string(); + let record = memory::extract::StagingRecord::from_candidate( + candidate_id, + request.operation.extract_run_id, + request.operation.source, + request.operation.candidate, + request.operation.evidence, + request.operation.source_refs, + ); + let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); + let (staged, _) = open_subjektiv_store(&api)? + .stage_candidate_with_attribution(record, attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + Ok(Json(server_api::SubjektivStageCandidateResponse { + staging_id: staged.id, + })) + }) + .await } async fn scoped_record_subjektiv_session( @@ -19972,25 +21105,34 @@ async fn scoped_record_subjektiv_session( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = if request.create_if_missing { - let attribution = - subjektiv_session_attribution(&api, &path.workspace_id, &context, &request.session_id)?; - open_subjektiv_store(&api)? - .record_session_attribution(attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))? - } else { - require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )? - }; - Ok(Json(server_api::SubjektivRecordSessionResponse { - subject_id: attribution.subject_id, - session_id: attribution.session_id, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = if request.create_if_missing { + let attribution = subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )?; + open_subjektiv_store(&api)? + .record_session_attribution(attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))? + } else { + require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )? + }; + Ok(Json(server_api::SubjektivRecordSessionResponse { + subject_id: attribution.subject_id, + session_id: attribution.session_id, + })) + }) + .await } const SUBJEKTIV_SESSION_DEFAULT_LIMIT: usize = 20; @@ -20084,50 +21226,61 @@ async fn scoped_subjektiv_session_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, controller, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - require_subjektiv_worker_authority( - authority, - SubjektivWorkerAuthority::Subject, - "Session history read", - )?; - let store = match open_subjektiv_store(&api) { - Ok(store) => store, - Err(error) => { - return Ok(Json(subjektiv_session_error( - server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, - error.error.to_string(), - ))); - } - }; - let response = match request.operation { - server_api::SubjektivSessionBackendOperation::List(input) => subjektiv_session_list( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - server_api::SubjektivSessionBackendOperation::Search(input) => subjektiv_session_search( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - server_api::SubjektivSessionBackendOperation::Read(input) => subjektiv_session_read( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - }; - Ok(Json(response)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, controller, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + require_subjektiv_worker_authority( + authority, + SubjektivWorkerAuthority::Subject, + "Session history read", + )?; + let store = match open_subjektiv_store(&api) { + Ok(store) => store, + Err(error) => { + return Ok(Json(subjektiv_session_error( + server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, + error.error.to_string(), + ))); + } + }; + let response = match request.operation { + server_api::SubjektivSessionBackendOperation::List(input) => { + subjektiv_session_list( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + server_api::SubjektivSessionBackendOperation::Search(input) => { + subjektiv_session_search( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + server_api::SubjektivSessionBackendOperation::Read(input) => { + subjektiv_session_read( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + }; + Ok(Json(response)) + }) + .await } fn subjektiv_session_list( @@ -21318,115 +22471,121 @@ async fn scoped_subjektiv_memory_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, source_worker, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - let store = open_subjektiv_store(&api)?; - use server_api::SubjektivMemoryBackendOperation as Op; - // Authentication, live Worker/Job authority and committed Session provenance - // remain Backend-owned. Only the domain operation/projection is shared. - let (expected, label) = match &request.operation { - Op::ResidentSummary(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "resident surface read", - ), - Op::ResidentContext(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "resident subject context read", - ), - Op::ValidateProposal(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "Memory proposal validation", - ), - Op::ReceiptStatus(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "explicit candidate receipt lookup", - ), - Op::StageExplicit(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "explicit candidate staging", - ), - Op::ListCandidates(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate listing", - ), - Op::ReadCandidate(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate read", - ), - Op::DecideCandidate(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate decision", - ), - Op::PrepareSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface generation preparation", - ), - Op::PublishSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface publication", - ), - Op::FailSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface generation failure", - ), - Op::Query(_) | Op::Read(_) | Op::ListChanges(_) => (None, "recall"), - }; - if let Some(expected) = expected { - require_subjektiv_worker_authority(authority, expected, label)?; - } - let host_context = match authority { - SubjektivWorkerAuthority::Subject => { - let attribution = if let Op::StageExplicit(input) = &request.operation { - Some(require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &input.session_id, - )?) - } else { - None + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, source_worker, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + let store = open_subjektiv_store(&api)?; + use server_api::SubjektivMemoryBackendOperation as Op; + // Authentication, live Worker/Job authority and committed Session provenance + // remain Backend-owned. Only the domain operation/projection is shared. + let (expected, label) = match &request.operation { + Op::ResidentSummary(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "resident surface read", + ), + Op::ResidentContext(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "resident subject context read", + ), + Op::ValidateProposal(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "Memory proposal validation", + ), + Op::ReceiptStatus(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "explicit candidate receipt lookup", + ), + Op::StageExplicit(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "explicit candidate staging", + ), + Op::ListCandidates(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate listing", + ), + Op::ReadCandidate(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate read", + ), + Op::DecideCandidate(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate decision", + ), + Op::PrepareSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface generation preparation", + ), + Op::PublishSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface publication", + ), + Op::FailSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface generation failure", + ), + Op::Query(_) | Op::Read(_) | Op::ListChanges(_) => (None, "recall"), }; - HostOperationContext::validated_body(&store, &subject_id, attribution)? - } - SubjektivWorkerAuthority::Consolidation => { - match &request.operation { - Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - _ => {} + if let Some(expected) = expected { + require_subjektiv_worker_authority(authority, expected, label)?; } - let grant = subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; - let job = api - .store - .worker_registry_projection(&path.workspace_id, &source_worker)? - .and_then(|projection| projection.job) - .map(|binding| JobAttemptBinding { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - }); - HostOperationContext::validated_consolidation( + let host_context = match authority { + SubjektivWorkerAuthority::Subject => { + let attribution = if let Op::StageExplicit(input) = &request.operation { + Some(require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &input.session_id, + )?) + } else { + None + }; + HostOperationContext::validated_body(&store, &subject_id, attribution)? + } + SubjektivWorkerAuthority::Consolidation => { + match &request.operation { + Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + _ => {} + } + let grant = + subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; + let job = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|projection| projection.job) + .map(|binding| JobAttemptBinding { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + }); + HostOperationContext::validated_consolidation( + &store, + &subject_id, + grant.map(|g| g.candidate_ids), + job, + )? + } + }; + Ok(Json(subjektiv::api::execute( &store, - &subject_id, - grant.map(|g| g.candidate_ids), - job, - )? - } - }; - Ok(Json(subjektiv::api::execute( - &store, - &host_context, - request.operation, - )?)) + &host_context, + request.operation, + )?)) + }) + .await } fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error { @@ -21530,10 +22689,17 @@ async fn start_subjektiv_staging_consolidation( subject_id: &str, operation: MemoryConsolidateStagingOperation, ) -> ApiResult { - let store = open_subjektiv_store(&api)?; - let (candidate_count, total_bytes) = store - .pending_staging_backlog(subject_id) - .map_err(subjektiv_store_error)?; + let job_api = api.clone(); + let subject = subject_id.to_owned(); + let (candidate_count, total_bytes) = api + .store + .db_executor() + .run(move || { + open_subjektiv_store(&job_api)? + .pending_staging_backlog(&subject) + .map_err(|error| ApiError::from(subjektiv_store_error(error))) + }) + .await?; if !subjektiv_job::retire_legacy_worker(&api, subject_id).await? { return Ok(MemoryConsolidationOutput { status: "skipped_legacy_inflight".into(), @@ -21541,10 +22707,18 @@ async fn start_subjektiv_staging_consolidation( candidate_count, total_bytes, }); } - let surface_availability = store - .resident_surface(subject_id) - .map_err(subjektiv_store_error)? - .availability; + let job_api = api.clone(); + let subject = subject_id.to_owned(); + let surface_availability = api + .store + .db_executor() + .run(move || { + open_subjektiv_store(&job_api)? + .resident_surface(&subject) + .map(|surface| surface.availability) + .map_err(|error| ApiError::from(subjektiv_store_error(error))) + }) + .await?; if !subject_consolidation_required( candidate_count, total_bytes, @@ -21571,7 +22745,11 @@ async fn start_subjektiv_staging_consolidation( }); } - subjektiv_job::dispatch(&api, subject_id, candidate_count, total_bytes) + let subject = subject_id.to_owned(); + api.host_executors + .host()? + .run(move || subjektiv_job::dispatch(&api, &subject, candidate_count, total_bytes)) + .await } async fn scoped_memory_consolidation( @@ -21579,8 +22757,13 @@ async fn scoped_memory_consolidation( AxumPath(path): AxumPath, Json(operation): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(start_memory_staging_consolidation(api, operation)?)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(start_memory_staging_consolidation(api, operation)?)) + }) + .await } fn start_memory_staging_consolidation( @@ -21780,72 +22963,92 @@ async fn scoped_list_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::catalog(&state).map(Json).map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::catalog(&state).map(Json).map_err(skill_api_error) + }) + .await } async fn scoped_lint_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::lint(&state).map(Json).map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::lint(&state).map(Json).map_err(skill_api_error) + }) + .await } async fn scoped_get_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::detail(&state, &path.name) - .map(Json) - .map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::detail(&state, &path.name) + .map(Json) + .map_err(skill_api_error) + }) + .await } async fn scoped_activate_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::activation(&state, &path.name) - .map(Json) - .map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::activation(&state, &path.name) + .map(Json) + .map_err(skill_api_error) + }) + .await } fn skill_api_error(error: skills::SkillError) -> ApiError { @@ -21895,8 +23098,13 @@ async fn scoped_query_objectives( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_objectives(query)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_objectives(query)?)) + }) + .await } async fn scoped_show_objective( @@ -21904,10 +23112,15 @@ async fn scoped_show_objective( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority.show_objective(&path.objective_id, query)?, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority.show_objective(&path.objective_id, query)?, + )) + }) + .await } async fn scoped_create_objective( @@ -21915,15 +23128,20 @@ async fn scoped_create_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.create_objective( - ObjectiveCreateInput { - title: request.title, - body_md: request.body_md, - state: request.state, - linked_tickets: request.linked_tickets, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.create_objective( + ObjectiveCreateInput { + title: request.title, + body_md: request.body_md, + state: request.state, + linked_tickets: request.linked_tickets, + }, + )?)) + }) + .await } async fn scoped_edit_objective( @@ -21931,16 +23149,21 @@ async fn scoped_edit_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.edit_objective( - &path.objective_id, - ObjectiveEditInput { - title: request.title, - old_string: request.old_string, - new_string: request.new_string, - replace_all: request.replace_all, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.edit_objective( + &path.objective_id, + ObjectiveEditInput { + title: request.title, + old_string: request.old_string, + new_string: request.new_string, + replace_all: request.replace_all, + }, + )?)) + }) + .await } async fn scoped_set_objective_state( @@ -21948,11 +23171,16 @@ async fn scoped_set_objective_state( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority - .set_objective_state(&path.objective_id, &request.state)?, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority + .set_objective_state(&path.objective_id, &request.state)?, + )) + }) + .await } async fn scoped_link_objective_ticket( @@ -21960,22 +23188,32 @@ async fn scoped_link_objective_ticket( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.link_objective_ticket( - &path.objective_id, - &request.ticket_id, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.link_objective_ticket( + &path.objective_id, + &request.ticket_id, + )?)) + }) + .await } async fn scoped_unlink_objective_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.unlink_objective_ticket( - &path.objective_id, - &path.ticket_id, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.unlink_objective_ticket( + &path.objective_id, + &path.ticket_id, + )?)) + }) + .await } async fn scoped_list_repositories( @@ -22483,6 +23721,9 @@ async fn worker_remove_contract( .map_err(|error| { let status = match error { crate::worker_source::WorkerMutationSourceProofError::Replay => StatusCode::CONFLICT, + crate::worker_source::WorkerMutationSourceProofError::Unavailable => { + StatusCode::SERVICE_UNAVAILABLE + } crate::worker_source::WorkerMutationSourceProofError::Authority(_) => { StatusCode::INTERNAL_SERVER_ERROR } @@ -22497,7 +23738,8 @@ async fn worker_remove_contract( ) })?; let worker = - WorkspaceWorker::resolve(api, &request.target_runtime_id, &request.target_worker_id) + WorkspaceWorker::resolve_async(api, &request.target_runtime_id, &request.target_worker_id) + .await .map_err(|error| match error { Error::UnknownWorker { .. } => server_api::WorkerRemoveApiError::new( StatusCode::NOT_FOUND.as_u16(), @@ -22553,22 +23795,28 @@ async fn scoped_get_workspace_worker( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker_ref = api - .store - .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? - .ok_or_else(|| Error::UnknownWorker { - worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), - })?; - let workers = workers_response(api.clone())?; - workers - .items - .into_iter() - .find(|worker| { - worker.runtime_id == worker_ref.runtime_id && worker.worker_id == worker_ref.worker_id + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let worker_ref = api + .store + .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? + .ok_or_else(|| Error::UnknownWorker { + worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), + })?; + let workers = workers_response(api.clone())?; + workers + .items + .into_iter() + .find(|worker| { + worker.runtime_id == worker_ref.runtime_id + && worker.worker_id == worker_ref.worker_id + }) + .map(Json) + .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) }) - .map(Json) - .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) + .await } async fn discover_workspace_workers( @@ -22691,8 +23939,13 @@ async fn scoped_workspace_orchestrator_status( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(workspace_orchestrator_response(&api, "observed"))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(workspace_orchestrator_response(&api, "observed"))) + }) + .await } async fn list_known_workers( @@ -22700,36 +23953,43 @@ async fn list_known_workers( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let limit = api.config.max_records.clamp(1, 500); - let grants = - api.store - .list_active_worker_control_grants(&path.workspace_id, &controller, limit + 1)?; - let truncated = grants.len() > limit; - let mut items = Vec::with_capacity(grants.len().min(limit)); - for grant in grants.into_iter().take(limit) { - let summary = api - .runtime - .worker(&grant.subject) - .map_err(|error| error.into_error())?; - items.push(server_api::WorkerControlRecord { - subject: server_api::WorkerControlSubject { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - relation: grant.relation, - origin: grant.origin, - permissions: grant.permissions, - summary: worker_launch_worker_summary(summary), - }); - } - Ok(Json(server_api::WorkerControlListResponse { - workspace_id: path.workspace_id, - items, - truncated, - })) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let limit = api.config.max_records.clamp(1, 500); + let grants = api.store.list_active_worker_control_grants( + &path.workspace_id, + &controller, + limit + 1, + )?; + let truncated = grants.len() > limit; + let mut items = Vec::with_capacity(grants.len().min(limit)); + for grant in grants.into_iter().take(limit) { + let summary = api + .runtime + .worker(&grant.subject) + .map_err(|error| error.into_error())?; + items.push(server_api::WorkerControlRecord { + subject: server_api::WorkerControlSubject { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + relation: grant.relation, + origin: grant.origin, + permissions: grant.permissions, + summary: worker_launch_worker_summary(summary), + }); + } + Ok(Json(server_api::WorkerControlListResponse { + workspace_id: path.workspace_id, + items, + truncated, + })) + }) + .await } fn scoped_worker_control_operation_id(controller: &RuntimeWorkerRef, operation_id: &str) -> String { @@ -22750,6 +24010,27 @@ async fn spawn_known_worker( } async fn spawn_known_worker_authenticated( + State(api): State, + AxumPath(path): AxumPath, + headers: HeaderMap, + Json(request): Json, + context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + spawn_known_worker_authenticated_on_host( + State(api), + AxumPath(path), + headers, + Json(request), + context.as_ref(), + ) + .await + }) + .await +} + +async fn spawn_known_worker_authenticated_on_host( State(api): State, AxumPath(path): AxumPath, headers: HeaderMap, @@ -22792,7 +24073,7 @@ async fn spawn_known_worker_authenticated( operation_id: scoped_worker_control_operation_id(&controller, &operation_id), input_fingerprint, }); - let response = create_workspace_worker_authenticated( + let response = create_workspace_worker_authenticated_on_host( api.clone(), headers, request, @@ -22880,7 +24161,9 @@ async fn send_known_worker_input( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; let request = worker_input_request_from_api(request)?; let idle_only = request.kind == WorkerInputKind::User; let result = worker.input(&context, request).await?; @@ -22916,7 +24199,9 @@ async fn cancel_known_worker( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; Ok(Json(worker_lifecycle_result_to_api( worker .cancel(&context, worker_lifecycle_request_from_api(request)) @@ -22935,7 +24220,9 @@ async fn stop_known_worker( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; Ok(Json(worker_lifecycle_result_to_api( worker .stop(&context, worker_lifecycle_request_from_api(request)) @@ -22969,41 +24256,46 @@ async fn scoped_list_worker_observation_sessions( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let sessions = api - .store - .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? - .into_iter() - .filter(|grant| { - grant - .permissions - .iter() - .any(|permission| permission == "observe") + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let sessions = api + .store + .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? + .into_iter() + .filter(|grant| { + grant + .permissions + .iter() + .any(|permission| permission == "observe") + }) + .filter_map(|grant| { + let worker = api.runtime.worker(&grant.subject).ok()?; + if matches!( + worker.state.as_str(), + "stopped" | "failed" | "rejected" | "disconnected" + ) { + return None; + } + Some(server_api::WorkerObservationSubject { + subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + display_name: worker.display_name, + relation: grant.relation, + status: worker.state, + }) + }) + .collect::>(); + Ok(Json(server_api::WorkerObservationSessionsResponse { + sessions, + })) }) - .filter_map(|grant| { - let worker = api.runtime.worker(&grant.subject).ok()?; - if matches!( - worker.state.as_str(), - "stopped" | "failed" | "rejected" | "disconnected" - ) { - return None; - } - Some(server_api::WorkerObservationSubject { - subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - display_name: worker.display_name, - relation: grant.relation, - status: worker.state, - }) - }) - .collect::>(); - Ok(Json(server_api::WorkerObservationSessionsResponse { - sessions, - })) + .await } async fn scoped_capture_worker_observation_session( @@ -23091,6 +24383,7 @@ async fn scoped_start_workspace_orchestrator( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { + api.host_executors.host()?.run(move || { #[cfg(test)] tests::seed_test_delivery_binding(&api); validate_workspace_scope(&api, &path.workspace_id)?; @@ -23217,6 +24510,8 @@ async fn scoped_start_workspace_orchestrator( drop(_guard); dispatch_orchestrator_assignment_attention(&api, &worker.worker); Ok(Json(workspace_orchestrator_response(&api, disposition))) + + }).await } fn worker_launch_worker_summary(worker: InternalWorkerSummary) -> WorkerLaunchWorkerSummary { @@ -23349,13 +24644,18 @@ async fn scoped_list_runtime_working_directories( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) + api.host_executors + .host()? + .run(move || { + let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) + }) + .await } async fn scoped_create_runtime_working_directory( @@ -23379,7 +24679,12 @@ async fn scoped_runtime_working_directory_detail( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) + api.host_executors + .host()? + .run(move || { + working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) + }) + .await } fn workdir_removal_source_actor( @@ -23465,19 +24770,25 @@ async fn cleanup_runtime_working_directory( ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - let registered_runtime = registered_workdir_runtime_id(&api, &path.working_directory_id)?; - if registered_runtime != path.runtime_id { - return Err(ApiError::from(Error::WorkspacePermissionDenied( - "Workdir does not belong to the requested Runtime".to_string(), - ))); - } - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) + api.host_executors + .host()? + .run(move || { + let registered_runtime = + registered_workdir_runtime_id(&api, &path.working_directory_id)?; + if registered_runtime != path.runtime_id { + return Err(ApiError::from(Error::WorkspacePermissionDenied( + "Workdir does not belong to the requested Runtime".to_string(), + ))); + } + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) + }) + .await } fn external_workdir_expires_at( @@ -23529,9 +24840,18 @@ fn schedule_external_workdir_expiry( } Err(error) => { tracing::error!(grant_id = %task_grant_id, error = ?error, "Backend External Workdir expiry cleanup requires retry"); + let stage_api = task_api.clone(); + let stage_grant_id = task_grant_id.clone(); let retryable = task_api .store - .get_external_workdir_grant(&task_api.config.workspace_id, &task_grant_id) + .db_executor() + .run(move || { + stage_api.store.get_external_workdir_grant( + &stage_api.config.workspace_id, + &stage_grant_id, + ) + }) + .await .map_or(true, |grant| grant.is_some()); if !retryable { break; @@ -23596,6 +24916,18 @@ async fn stop_external_workdir_provider( async fn fence_external_workdir_grant_expired( api: &WorkspaceApi, grant_id: &str, +) -> ApiResult { + let stage_api = api.clone(); + let grant_id = grant_id.to_string(); + api.store + .db_executor() + .run(move || fence_external_workdir_grant_expired_stage(&stage_api, &grant_id)) + .await +} + +fn fence_external_workdir_grant_expired_stage( + api: &WorkspaceApi, + grant_id: &str, ) -> ApiResult { loop { let mut grant = api @@ -23665,6 +24997,8 @@ async fn scoped_create_external_workdir_grant( Extension(actor): Extension, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { + api.store.db_executor().run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; let display_name = request.display_name.trim(); if display_name.is_empty() @@ -23734,6 +25068,8 @@ async fn scoped_create_external_workdir_grant( StatusCode::CREATED, Json(external_workdir_grant_response(&grant)?), )) + + }).await } fn external_workdir_grant_response( @@ -23758,16 +25094,21 @@ async fn scoped_external_workdir_provider_ws( AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ws: WebSocketUpgrade, ) -> ApiResult { - validate_workspace_scope(&api, &workspace_id)?; - let mut grant = api - .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir provider connection must use the grant creator's authenticated client".to_string(), - ).into()); - } + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); + let mut grant = api.store.db_executor().run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &stage_workspace_id)?; + let grant = stage_api.store + .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir provider connection must use the grant creator's authenticated client".to_string(), + ).into()); + } + Ok(grant) + }).await?; let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(&api, &grant_id).await?; @@ -23914,81 +25255,102 @@ async fn serve_external_workdir_provider( .await; return; } - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !api + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let registration_generation = registration.generation; + let registration_capabilities = registration.capabilities; + let activation = api .store - .activate_external_workdir_grant( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration.generation, - &now, - ) - .unwrap_or(false) - { - let _ = socket.close().await; - return; - } - if let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_connected", - &initial_grant.grant_id, - None, - "online", - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, registration.generation - ), - &now, - ) { - let _ = api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration.generation, - "offline", - &now, - ); - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = registration.generation, - %error, - "failed to audit External Workdir provider activation" - ); - let _ = socket.close().await; - return; - } - let expires_at = match external_workdir_expires_at(&initial_grant) { - Ok(value) => value, - Err(_) => { + .db_executor() + .run(move || -> Result<_> { + let api = stage_api; + let initial_grant = stage_grant; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !api + .store + .activate_external_workdir_grant( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration_generation, + &now, + ) + .unwrap_or(false) + { + return Ok(None); + } + if let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_connected", + &initial_grant.grant_id, + None, + "online", + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, registration_generation + ), + &now, + ) { + let _ = api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration_generation, + "offline", + &now, + ); + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = registration_generation, + %error, + "failed to audit External Workdir provider activation" + ); + return Ok(None); + } + let expires_at = match external_workdir_expires_at(&initial_grant) { + Ok(value) => value, + Err(_) => return Ok(None), + }; + let (sender, commands) = tokio::sync::mpsc::channel(32); + let connection = Arc::new(ExternalProviderConnection { + grant_id: initial_grant.grant_id.clone(), + workdir_id: initial_grant.workdir_id.clone(), + provider_instance_id: initial_grant.provider_instance_id.clone(), + generation: registration_generation, + expires_at, + capabilities: registration_capabilities, + admission: Arc::new(tokio::sync::Semaphore::new(16)), + shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + sender, + }); + // Publish with the accepted activation stage, never in its cancellable waiter. + if api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connected(connection.clone()) + .is_some() + { + return Ok(None); + } + Ok(Some((connection, commands))) + }) + .await; + let (connection, mut commands) = match activation { + Ok(Some(activated)) => activated, + Ok(None) => { + let _ = socket.close().await; + return; + } + Err(error) => { + tracing::error!(grant_id = %initial_grant.grant_id, %error, + "External Workdir provider activation stage failed"); let _ = socket.close().await; return; } }; - let (sender, mut commands) = tokio::sync::mpsc::channel(32); - let connection = Arc::new(ExternalProviderConnection { - grant_id: initial_grant.grant_id.clone(), - workdir_id: initial_grant.workdir_id.clone(), - provider_instance_id: initial_grant.provider_instance_id.clone(), - generation: registration.generation, - expires_at, - capabilities: registration.capabilities, - admission: Arc::new(tokio::sync::Semaphore::new(16)), - shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), - sender, - }); - if api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connected(connection.clone()) - .is_some() - { - let _ = socket.close().await; - return; - } let registered = ExternalWorkdirServerFrame::current(ExternalWorkdirServerMessage::Registered { generation: connection.generation, @@ -23996,19 +25358,36 @@ async fn serve_external_workdir_provider( max_in_flight_operations: 16, }); if !send_external_workdir_server_frame(&mut socket, ®istered).await { - let _ = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - if let Err(error) = api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &connection.provider_instance_id, - connection.generation, - "offline", - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let stage_connection = connection.clone(); + if let Err(error) = api + .store + .db_executor() + .run(move || { + stage_api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&stage_connection.grant_id, stage_connection.generation); + stage_api.store.update_external_workdir_grant_state( + &stage_grant.workspace_id, + &stage_grant.grant_id, + &stage_connection.provider_instance_id, + stage_connection.generation, + "offline", + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await + { + // Rejected admission must not leave a dead socket discoverable. + // The generation-bound in-memory fence is safe even for an unknown + // DB outcome; it neither guesses nor retries the durable mutation. + api.external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); tracing::error!( grant_id = %initial_grant.grant_id, generation = connection.generation, @@ -24158,55 +25537,81 @@ async fn serve_external_workdir_provider( } }; pending.clear(); - let _ = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - let state_updated = match api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &connection.provider_instance_id, - connection.generation, - terminal_status, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let stage_connection = connection.clone(); + let terminal_update = api + .store + .db_executor() + .run(move || -> Result<_> { + let api = stage_api; + let initial_grant = stage_grant; + let connection = stage_connection; + let _ = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + let state_updated = match api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &connection.provider_instance_id, + connection.generation, + terminal_status, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) { + Ok(updated) => updated, + Err(error) => { + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to persist External Workdir provider terminal state" + ); + false + } + }; + if state_updated + && let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_disconnected", + &initial_grant.grant_id, + None, + terminal_status, + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + { + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to audit External Workdir provider terminal state" + ); + } + Ok(state_updated) + }) + .await; + let state_updated = match terminal_update { Ok(updated) => updated, Err(error) => { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to persist External Workdir provider terminal state" - ); + api.external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + tracing::error!(grant_id = %initial_grant.grant_id, + generation = connection.generation, status = terminal_status, %error, + "External Workdir provider terminal stage failed"); false } }; - if state_updated - && let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_disconnected", - &initial_grant.grant_id, - None, - terminal_status, - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to audit External Workdir provider terminal state" - ); - } if state_updated && matches!(terminal_status, "expired" | "revoked") && let Err(error) = @@ -24226,17 +25631,27 @@ async fn scoped_get_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); let mut grant = api .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir grant inspection requires the grant creator".to_string(), - ) - .into()); - } + .db_executor() + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &stage_workspace_id)?; + let grant = stage_api + .store + .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir grant inspection requires the grant creator".to_string(), + ) + .into()); + } + Ok(grant) + }) + .await?; let expired = !matches!(grant.status.as_str(), "revoked" | "expired") && external_workdir_grant_is_expired(&grant)?; if expired { @@ -24263,44 +25678,57 @@ async fn scoped_revoke_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let mut grant = api + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); + let grant = api .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir revoke requires the grant creator".to_string(), - ) - .into()); - } - let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !matches!(grant.status.as_str(), "revoked" | "expired") { - api.store.update_external_workdir_grant_state( - &workspace_id, - &grant_id, - &grant.provider_instance_id, - grant.generation, - "revoked", - &updated_at, - )?; - grant.status = "revoked".to_string(); - grant.updated_at = updated_at; - api.store.record_external_workdir_audit( - &workspace_id, - "account", - &actor.account_id, - "external_workdir_grant_revoked", - &grant_id, - None, - "revoked", - &format!( - "workdir_id={} generation={}", - grant.workdir_id, grant.generation - ), - &grant.updated_at, - )?; - } + .db_executor() + .run(move || -> ApiResult<_> { + let api = stage_api; + let workspace_id = stage_workspace_id; + let grant_id = stage_grant_id; + validate_workspace_scope(&api, &workspace_id)?; + let mut grant = api + .store + .get_external_workdir_grant(&workspace_id, &grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir revoke requires the grant creator".to_string(), + ) + .into()); + } + let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !matches!(grant.status.as_str(), "revoked" | "expired") { + api.store.update_external_workdir_grant_state( + &workspace_id, + &grant_id, + &grant.provider_instance_id, + grant.generation, + "revoked", + &updated_at, + )?; + grant.status = "revoked".to_string(); + grant.updated_at = updated_at; + api.store.record_external_workdir_audit( + &workspace_id, + "account", + &actor.account_id, + "external_workdir_grant_revoked", + &grant_id, + None, + "revoked", + &format!( + "workdir_id={} generation={}", + grant.workdir_id, grant.generation + ), + &grant.updated_at, + )?; + } + Ok(grant) + }) + .await?; // Cleanup is deliberately retried for terminal grants. A prior revoke may // have committed its fence before a Runtime/session projection failure. let cleanup = cleanup_external_workdir_attachments(&api, &grant.workdir_id).await; @@ -24325,70 +25753,88 @@ async fn cleanup_external_workdir_attachments( async fn cleanup_external_workdir_attachments_with_lock( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<&RuntimeWorkerRef>, + already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, ) -> ApiResult<()> { - let workdir = api + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); + let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); + let grant_id = api .store - .get_workdir_registry(&api.config.workspace_id, workdir_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; - let grant_id = workdir.source.external_grant_id().ok_or_else(|| { - Error::InvalidInput("Workdir is not backed by an External grant".to_string()) - })?; - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "pending", - None, - &now, - )?; - - match release_external_workdir_attachments(api, workdir_id, already_locked_worker).await { - Ok(()) => { - let completed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "completed", + .db_executor() + .run(move || -> Result<_> { + let _session_guard = stage_guard; + let workdir = stage_api + .store + .get_workdir_registry(&stage_api.config.workspace_id, &stage_workdir_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; + let grant_id = workdir + .source + .external_grant_id() + .ok_or_else(|| { + Error::InvalidInput("Workdir is not backed by an External grant".to_string()) + })? + .to_string(); + stage_api.store.update_external_workdir_cleanup_state( + &stage_api.config.workspace_id, + &grant_id, + "pending", None, - &completed_at, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), )?; - api.store.record_external_workdir_audit( - &api.config.workspace_id, + Ok(grant_id) + }) + .await?; + + let release = + release_external_workdir_attachments(api, workdir_id, already_locked_worker).await; + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); + let succeeded = release.is_ok(); + let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); + api.store + .db_executor() + .run(move || -> Result<()> { + let _session_guard = stage_guard; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let (state, category, event, outcome, detail) = if succeeded { + ( + "completed", + None, + "external_workdir_cleanup_completed", + "completed", + format!("workdir_id={stage_workdir_id}"), + ) + } else { + ( + "retry_required", + Some("attachment_release_failed"), + "external_workdir_cleanup_attention_required", + "attention_required", + format!("workdir_id={stage_workdir_id} category=attachment_release_failed"), + ) + }; + stage_api.store.update_external_workdir_cleanup_state( + &stage_api.config.workspace_id, + &grant_id, + state, + category, + &now, + )?; + stage_api.store.record_external_workdir_audit( + &stage_api.config.workspace_id, "system", "workspace-backend", - "external_workdir_cleanup_completed", - grant_id, + event, + &grant_id, None, - "completed", - &format!("workdir_id={workdir_id}"), - &completed_at, + outcome, + &detail, + &now, )?; Ok(()) - } - Err(error) => { - let failed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "retry_required", - Some("attachment_release_failed"), - &failed_at, - )?; - api.store.record_external_workdir_audit( - &api.config.workspace_id, - "system", - "workspace-backend", - "external_workdir_cleanup_attention_required", - grant_id, - None, - "attention_required", - &format!("workdir_id={workdir_id} category=attachment_release_failed"), - &failed_at, - )?; - Err(error) - } - } + }) + .await?; + release } #[cfg(test)] @@ -24449,17 +25895,28 @@ async fn release_external_workdir_attachment_locked( api: &WorkspaceApi, workdir_id: &str, link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, ) -> ApiResult<()> { if link.unlinked_at.is_none() { close_current_worker_attachment_session_locked(api, &link.worker).await?; close_current_worker_command_sessions_for_alias_locked(api, &link.worker, &link.alias) .await?; - api.store.detach_worker_workdir( - &api.config.workspace_id, - &link.worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; + let stage_api = api.clone(); + let stage_worker = link.worker.clone(); + let stage_workdir_id = workdir_id.to_string(); + let stage_guard = session_guard.clone(); + api.store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + stage_api.store.detach_worker_workdir( + &stage_api.config.workspace_id, + &stage_worker, + Some(&stage_workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await?; #[cfg(test)] if take_external_workdir_cleanup_failure(workdir_id) { return Err(Error::RuntimeOperationFailed { @@ -24474,9 +25931,10 @@ async fn release_external_workdir_attachment_locked( // have committed detach before Runtime persistence or projection failed. #[cfg(test)] record_external_workdir_cleanup_reconciliation(workdir_id); - sync_runtime_worker_workdir_attachments(api, &link.worker)?; + sync_runtime_worker_workdir_attachments_locked(api, &link.worker, session_guard).await?; api.worker_projection - .refresh(&link.worker) + .refresh_async(&link.worker) + .await .map_err(ApiError::from)?; Ok(()) } @@ -24484,23 +25942,34 @@ async fn release_external_workdir_attachment_locked( async fn release_external_workdir_attachments( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<&RuntimeWorkerRef>, + already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, ) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); let Some(link) = api .store - .latest_workdir_worker_link(&api.config.workspace_id, workdir_id)? + .db_executor() + .run(move || { + stage_api + .store + .latest_workdir_worker_link(&stage_api.config.workspace_id, &stage_workdir_id) + }) + .await? else { return Ok(()); }; // The newest link is the only possible active attachment and remains the // durable reconciliation target if detach committed before Runtime update. // Missing/ambiguous Runtime state remains a retryable cleanup failure. - if already_locked_worker == Some(&link.worker) { - release_external_workdir_attachment_locked(api, workdir_id, &link).await?; + if let Some((worker, guard)) = + already_locked_worker.filter(|(worker, _)| *worker == &link.worker) + { + debug_assert_eq!(worker, &link.worker); + release_external_workdir_attachment_locked(api, workdir_id, &link, guard).await?; } else { let lock = current_worker_session_lock(api, &link.worker); - let _guard = lock.lock().await; - release_external_workdir_attachment_locked(api, workdir_id, &link).await?; + let guard = Arc::new(lock.lock_owned().await); + release_external_workdir_attachment_locked(api, workdir_id, &link, &guard).await?; } Ok(()) } @@ -24509,14 +25978,19 @@ async fn scoped_list_working_directories( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (items, diagnostics) = working_directory_summaries(&api)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (items, diagnostics) = working_directory_summaries(&api)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) + }) + .await } async fn scoped_create_working_directory( @@ -24531,40 +26005,47 @@ async fn scoped_working_directory_detail( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let record = api - .store - .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "working_directory_not_found".to_string(), - message: format!("Unknown Workdir `{}`", path.working_directory_id), - })?; - if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { - let grant = api - .store - .get_workspace_config_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| { - Error::WorkspacePermissionDenied("Workspace config grant unavailable".into()) - })?; - return Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: workspace_config::summary(&grant, &record), - diagnostics: vec![], - })); - } - if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { - working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) - } else { - let item = projected_workdir_summary_from_record(&api, &record)?; - Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: item.into(), - diagnostics: Vec::new(), - })) - } + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let record = api + .store + .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "working_directory_not_found".to_string(), + message: format!("Unknown Workdir `{}`", path.working_directory_id), + })?; + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Workspace config grant unavailable".into(), + ) + })?; + return Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: workspace_config::summary(&grant, &record), + diagnostics: vec![], + })); + } + if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { + working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) + } else { + let item = projected_workdir_summary_from_record(&api, &record)?; + Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: item.into(), + diagnostics: Vec::new(), + })) + } + }) + .await } async fn cleanup_working_directory( @@ -24574,13 +26055,18 @@ async fn cleanup_working_directory( request: WorkingDirectoryRemovalRequest, ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) + api.host_executors + .host()? + .run(move || { + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) + }) + .await } fn registered_workdir_runtime_id( @@ -24652,12 +26138,49 @@ fn finish_rejected_workdir_create_operation( Ok(failure_code) } -async fn create_workspace_working_directory( +async fn fail_workdir_create_stage( + api: &WorkspaceApi, + reserved: &WorkdirCreateOperationRecord, + request_fingerprint: &str, + failure: &str, + remove_registry: bool, +) -> Result { + let stage_api = api.clone(); + let reserved = reserved.clone(); + let request_fingerprint = request_fingerprint.to_string(); + let failure = failure.to_string(); + api.store + .db_executor() + .run(move || { + if remove_registry { + let _ = stage_api.store.delete_workdir_registry( + &reserved.workspace_id, + &reserved.working_directory_id, + ); + } + stage_api.config_store.finish_workdir_create_operation( + &reserved.workspace_id, + &reserved.operation_id, + &request_fingerprint, + false, + Some(&failure), + &now_registry_timestamp(), + ) + }) + .await +} + +fn prepare_workspace_workdir_create( api: &WorkspaceApi, workspace_id: &str, route_runtime_id: Option<&str>, mut request: BrowserWorkingDirectoryCreateRequest, -) -> ApiResult<(StatusCode, Json)> { +) -> ApiResult<( + WorkingDirectoryRequest, + String, + String, + WorkdirCreateOperationRecord, +)> { validate_workspace_scope(api, workspace_id)?; request.display_name = request .display_name @@ -24816,55 +26339,111 @@ async fn create_workspace_working_directory( })? }; - if reserved.state == "succeeded" { - return working_directory_detail_for_runtime( - api.clone(), - &reserved.resolved_runtime_id, - &reserved.working_directory_id, - ) - .map(|Json(response)| { - ( - StatusCode::OK, - Json(BrowserWorkingDirectoryCreateResponse { - workspace_id: response.workspace_id, - runtime_id: response - .runtime_id - .expect("Repository Workdir detail has Runtime"), - item: response.item, - diagnostics: response.diagnostics, - http_status: StatusCode::OK.as_u16(), - }), + Ok(( + working_directory_request, + operation_id, + request_fingerprint, + reserved, + )) +} + +async fn create_workspace_working_directory( + api: &WorkspaceApi, + workspace_id: &str, + route_runtime_id: Option<&str>, + request: BrowserWorkingDirectoryCreateRequest, +) -> ApiResult<(StatusCode, Json)> { + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.to_string(); + let route_runtime_id = route_runtime_id.map(str::to_string); + let (mut working_directory_request, operation_id, request_fingerprint, reserved) = api + .store + .db_executor() + .run(move || { + prepare_workspace_workdir_create( + &stage_api, + &stage_workspace_id, + route_runtime_id.as_deref(), + request, ) - }); + }) + .await?; + + if reserved.state == "succeeded" { + let stage_api = api.clone(); + return api + .host_executors + .host()? + .run(move || { + working_directory_detail_for_runtime( + stage_api, + &reserved.resolved_runtime_id, + &reserved.working_directory_id, + ) + .map(|Json(response)| { + ( + StatusCode::OK, + Json(BrowserWorkingDirectoryCreateResponse { + workspace_id: response.workspace_id, + runtime_id: response + .runtime_id + .expect("Repository Workdir detail has Runtime"), + item: response.item, + diagnostics: response.diagnostics, + http_status: StatusCode::OK.as_u16(), + }), + ) + }) + }) + .await; } let reserved = if reserved.state == "failed" { - api.config_store.begin_failed_workdir_create_retry( - workspace_id, - &operation_id, - &request_fingerprint, - &now_registry_timestamp(), - )? + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.to_string(); + let stage_operation_id = operation_id.clone(); + let stage_fingerprint = request_fingerprint.clone(); + api.store + .db_executor() + .run(move || { + stage_api.config_store.begin_failed_workdir_create_retry( + &stage_workspace_id, + &stage_operation_id, + &stage_fingerprint, + &now_registry_timestamp(), + ) + }) + .await? } else { reserved }; + let stage_api = api.clone(); + let stage_runtime_id = reserved.resolved_runtime_id.clone(); let runtime = match api - .runtime - .list_runtimes(usize::MAX) - .items - .into_iter() - .find(|runtime| runtime.runtime_id == reserved.resolved_runtime_id) + .host_executors + .host()? + .run(move || { + Ok::<_, Error>( + stage_api + .runtime + .list_runtimes(usize::MAX) + .items + .into_iter() + .find(|runtime| runtime.runtime_id == stage_runtime_id), + ) + }) + .await? { Some(runtime) => runtime, None => { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "runtime_unavailable", false, - Some("runtime_unavailable"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(ApiError::with_diagnostics( Error::UnknownRuntime(reserved.resolved_runtime_id), vec![RuntimeDiagnostic { @@ -24876,14 +26455,14 @@ async fn create_workspace_working_directory( } }; if runtime.kind == "remote_http" && runtime.status != "connected" { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "runtime_unavailable", false, - Some("runtime_unavailable"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: reserved.resolved_runtime_id, @@ -24898,38 +26477,52 @@ async fn create_workspace_working_directory( )); } - if let Err(error) = authorize_repository_materialization_operation( - api, - &reserved, - &request_fingerprint, - &mut working_directory_request, - ) { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let stage_fingerprint = request_fingerprint.clone(); + let (authorization, mut working_directory_request) = api + .store + .db_executor() + .run(move || { + let authorization = authorize_repository_materialization_operation( + &stage_api, + &stage_reserved, + &stage_fingerprint, + &mut working_directory_request, + ); + Ok::<_, Error>((authorization, working_directory_request)) + }) + .await?; + if let Err(error) = authorization { + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "working_directory_remote_repository_access_required", false, - Some("working_directory_remote_repository_access_required"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(error); } working_directory_request.backend_workdir_id = Some(reserved.working_directory_id.clone()); - let existing = match api.runtime.working_directory( - &reserved.resolved_runtime_id, - &reserved.working_directory_id, - ) { + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let existing = match api + .host_executors + .host()? + .run(move || { + Ok::<_, Error>(stage_api.runtime.working_directory( + &stage_reserved.resolved_runtime_id, + &stage_reserved.working_directory_id, + )) + }) + .await? + { Ok(existing) => existing, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, - &request_fingerprint, - false, - Some(&failure_code), - &now_registry_timestamp(), - )?; + fail_workdir_create_stage(api, &reserved, &request_fingerprint, &failure_code, false) + .await?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -24943,24 +26536,31 @@ async fn create_workspace_working_directory( let result = if existing.working_directory.is_some() { existing } else { + let stage_api = api.clone(); + let stage_runtime_id = reserved.resolved_runtime_id.clone(); match api - .runtime - .create_working_directory(&reserved.resolved_runtime_id, working_directory_request) + .host_executors + .host()? + .run(move || { + Ok::<_, Error>( + stage_api + .runtime + .create_working_directory(&stage_runtime_id, working_directory_request), + ) + }) + .await? { Ok(result) => result, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - let _ = api - .store - .delete_workdir_registry(workspace_id, &reserved.working_directory_id); - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, - false, - Some(&failure_code), - &now_registry_timestamp(), - )?; + &failure_code, + true, + ) + .await?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -24973,18 +26573,30 @@ async fn create_workspace_working_directory( } }; let Some(working_directory) = result.working_directory else { - let _ = api - .store - .delete_workdir_registry(workspace_id, &reserved.working_directory_id); let mut diagnostics = result.diagnostics; - let failure_code = finish_rejected_workdir_create_operation( - &api.config_store, - workspace_id, - &operation_id, - &request_fingerprint, - &diagnostics, - &now_registry_timestamp(), - )?; + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let stage_fingerprint = request_fingerprint.clone(); + let (failure_code, returned_diagnostics) = api + .store + .db_executor() + .run(move || -> Result<_> { + let _ = stage_api.store.delete_workdir_registry( + &stage_reserved.workspace_id, + &stage_reserved.working_directory_id, + ); + let failure_code = finish_rejected_workdir_create_operation( + &stage_api.config_store, + &stage_reserved.workspace_id, + &stage_reserved.operation_id, + &stage_fingerprint, + &diagnostics, + &now_registry_timestamp(), + )?; + Ok((failure_code, diagnostics)) + }) + .await?; + diagnostics = returned_diagnostics; if !diagnostics.iter().any(|diagnostic| { diagnostic.severity == HostDiagnosticSeverity::Error && diagnostic.code == failure_code }) { @@ -25006,21 +26618,29 @@ async fn create_workspace_working_directory( diagnostics, )); }; - let record = workdir_record_from_summary( - api, - &reserved.resolved_runtime_id, - &working_directory.summary, - ); - api.store.upsert_workdir_registry(&record)?; - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, - &request_fingerprint, - true, - None, - &now_registry_timestamp(), - )?; - let summary = projected_workdir_summary_from_record(api, &record)?; + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let summary = api + .store + .db_executor() + .run(move || -> ApiResult<_> { + let record = workdir_record_from_summary( + &stage_api, + &stage_reserved.resolved_runtime_id, + &working_directory.summary, + ); + stage_api.store.upsert_workdir_registry(&record)?; + stage_api.config_store.finish_workdir_create_operation( + &stage_reserved.workspace_id, + &stage_reserved.operation_id, + &request_fingerprint, + true, + None, + &now_registry_timestamp(), + )?; + Ok(projected_workdir_summary_from_record(&stage_api, &record)?) + }) + .await?; Ok(( StatusCode::CREATED, Json(BrowserWorkingDirectoryCreateResponse { @@ -25600,7 +27220,7 @@ async fn set_worker_retention( runtime_worker_id: String, pinned: bool, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &runtime_worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &runtime_worker_id).await?; Ok(Json( worker .set_pinned(&WorkerOperationContext::Backend, pinned) @@ -25864,7 +27484,13 @@ async fn execute_runtime_cleanup_with_context( } context.authorize_workspace(api).await?; - let plan = build_runtime_cleanup_plan(api, runtime_id)?; + let plan_api = api.clone(); + let plan_runtime = runtime_id.to_owned(); + let plan = api + .host_executors + .host()? + .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) + .await?; // Rebuild the actual candidate set immediately before executing. The digest // binds targets, dependencies and blockers; per-target authority is checked again below. if request.expected_plan_digest != plan.digest { @@ -25887,20 +27513,36 @@ async fn execute_runtime_cleanup_with_context( candidate.runtime_id.clone(), candidate.runtime_worker_id.clone(), ); - if let Some(claim) = api + let authority_api = api.clone(); + let target = worker.clone(); + let (claim, assignment) = api .store - .get_pending_ticket_assignment_operation_for_worker(&api.config.workspace_id, &worker)? - { + .db_executor() + .run(move || -> Result<_> { + Ok(( + authority_api + .store + .get_pending_ticket_assignment_operation_for_worker( + &authority_api.config.workspace_id, + &target, + )?, + authority_api + .store + .get_active_ticket_role_assignment_for_worker( + &authority_api.config.workspace_id, + &target, + )?, + )) + }) + .await?; + if let Some(claim) = claim { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_claim_pending", &format!("Worker has pending Ticket claim {}", claim.operation_id), )); } - if let Some(assignment) = api - .store - .get_active_ticket_role_assignment_for_worker(&api.config.workspace_id, &worker)? - { + if let Some(assignment) = assignment { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_assigned", @@ -25926,7 +27568,8 @@ async fn execute_runtime_cleanup_with_context( )); } parse_runtime_worker_id_for_registry(&candidate.runtime_worker_id)?; - WorkspaceWorker::resolve(api, &worker.runtime_id, &worker.worker_id)? + WorkspaceWorker::resolve_async(api, &worker.runtime_id, &worker.worker_id) + .await? .delete_from_plan(context, candidate) .await?; api.workdir_session_locks @@ -25965,12 +27608,21 @@ async fn execute_runtime_cleanup_with_context( }); } CleanupTargetKind::WorkdirCleanCleanup | CleanupTargetKind::WorkdirRecordDelete => { - let removal = execute_workdir_removal( - api, - candidate.workdir_id.as_str(), - &format!("runtime-cleanup:{runtime_id}"), - &format!("cleanup target {}", candidate.target_id), - )?; + let removal_api = api.clone(); + let removal_candidate = candidate.clone(); + let requester = format!("runtime-cleanup:{runtime_id}"); + let removal = api + .host_executors + .host()? + .run(move || { + execute_workdir_removal( + &removal_api, + removal_candidate.workdir_id.as_str(), + &requester, + &format!("cleanup target {}", removal_candidate.target_id), + ) + }) + .await?; let (status, message) = match removal.disposition { WorkingDirectoryRemovalDisposition::Removed => ( "deleted", @@ -26016,7 +27668,13 @@ async fn execute_runtime_cleanup_with_context( )); } - let plan_after = build_runtime_cleanup_plan(api, runtime_id)?; + let plan_api = api.clone(); + let plan_runtime = runtime_id.to_owned(); + let plan_after = api + .host_executors + .host()? + .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) + .await?; let executed_at = now_registry_timestamp(); Ok(RuntimeCleanupExecutionResponse { workspace_id: api.config.workspace_id.clone(), @@ -26289,7 +27947,9 @@ async fn scoped_restore_runtime_worker_with_context( request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let resolved = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; let _authorization = resolved.authorize_operation(&context, "restore").await?; let runtime_id = resolved.identity().runtime_id.clone(); let worker_id = resolved.identity().worker_id.clone(); @@ -26322,9 +27982,14 @@ async fn scoped_runtime_cleanup_plan( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; - Ok(Json(plan)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; + Ok(Json(plan)) + }) + .await } #[cfg(test)] @@ -26369,56 +28034,65 @@ async fn scoped_create_attachment_upload_grant( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - resolve_workspace_worker_reference(&api, &path.worker.runtime_id, &path.worker.worker_id)?; - let now = attachment_upload_now_ms()?; - let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); - let upload_id = match query.upload_id { - Some(upload_id) => Uuid::parse_str(&upload_id) - .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? - .to_string(), - None => Uuid::now_v7().to_string(), - }; - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - grants.retain(|_, grant| grant.expires_at_ms >= now); - if let Some(existing) = grants.get(&upload_id) { - if existing.workspace_id == path.workspace_id - && existing.runtime_id == path.worker.runtime_id - && existing.worker_id == path.worker.worker_id - && existing.account_id == actor.account_id - && existing.file_name == query.file_name - && existing.media_type == query.media_type - { - return Ok(Json(AttachmentUploadGrantResponse { + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + resolve_workspace_worker_reference( + &api, + &path.worker.runtime_id, + &path.worker.worker_id, + )?; + let now = attachment_upload_now_ms()?; + let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); + let upload_id = match query.upload_id { + Some(upload_id) => Uuid::parse_str(&upload_id) + .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? + .to_string(), + None => Uuid::now_v7().to_string(), + }; + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + grants.retain(|_, grant| grant.expires_at_ms >= now); + if let Some(existing) = grants.get(&upload_id) { + if existing.workspace_id == path.workspace_id + && existing.runtime_id == path.worker.runtime_id + && existing.worker_id == path.worker.worker_id + && existing.account_id == actor.account_id + && existing.file_name == query.file_name + && existing.media_type == query.media_type + { + return Ok(Json(AttachmentUploadGrantResponse { + upload_id, + expires_at_ms: existing.expires_at_ms, + })); + } + return Err(Error::RepositoryConflict( + "attachment upload id was reused with different intent".into(), + ) + .into()); + } + grants.insert( + upload_id.clone(), + AttachmentUploadGrant { + workspace_id: path.workspace_id, + runtime_id: path.worker.runtime_id, + worker_id: path.worker.worker_id, + account_id: actor.account_id, + file_name: query.file_name, + media_type: query.media_type, + expires_at_ms, + state: AttachmentUploadGrantState::Pending, + }, + ); + Ok(Json(AttachmentUploadGrantResponse { upload_id, - expires_at_ms: existing.expires_at_ms, - })); - } - return Err(Error::RepositoryConflict( - "attachment upload id was reused with different intent".into(), - ) - .into()); - } - grants.insert( - upload_id.clone(), - AttachmentUploadGrant { - workspace_id: path.workspace_id, - runtime_id: path.worker.runtime_id, - worker_id: path.worker.worker_id, - account_id: actor.account_id, - file_name: query.file_name, - media_type: query.media_type, - expires_at_ms, - state: AttachmentUploadGrantState::Pending, - }, - ); - Ok(Json(AttachmentUploadGrantResponse { - upload_id, - expires_at_ms, - })) + expires_at_ms, + })) + }) + .await } async fn upload_runtime_worker_file_contract( @@ -26503,49 +28177,55 @@ async fn scoped_cancel_attachment_upload( Extension(actor): Extension, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let completed = { - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - let grant = grants - .get_mut(&path.upload_id) - .ok_or_else(|| Error::InvalidInput("attachment upload grant is unknown".into()))?; - if grant.workspace_id != path.workspace_id - || grant.runtime_id != path.runtime_id - || grant.worker_id != path.worker_id - || grant.account_id != actor.account_id - { - return Err(Error::WorkspacePermissionDenied( - "attachment upload grant scope mismatch".into(), - ) - .into()); - } - match &grant.state { - AttachmentUploadGrantState::Pending => { - grants.remove(&path.upload_id); - None + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let completed = { + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + let grant = grants.get_mut(&path.upload_id).ok_or_else(|| { + Error::InvalidInput("attachment upload grant is unknown".into()) + })?; + if grant.workspace_id != path.workspace_id + || grant.runtime_id != path.runtime_id + || grant.worker_id != path.worker_id + || grant.account_id != actor.account_id + { + return Err(Error::WorkspacePermissionDenied( + "attachment upload grant scope mismatch".into(), + ) + .into()); + } + match &grant.state { + AttachmentUploadGrantState::Pending => { + grants.remove(&path.upload_id); + None + } + AttachmentUploadGrantState::Uploading => { + grant.state = AttachmentUploadGrantState::Cancelled; + None + } + AttachmentUploadGrantState::Cancelled => None, + AttachmentUploadGrantState::Completed(file) => { + let file = file.clone(); + grants.remove(&path.upload_id); + Some(file) + } + } + }; + if let Some(file) = completed { + let worker = + resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &file.artifact_id) + .map_err(|error| error.into_error())?; } - AttachmentUploadGrantState::Uploading => { - grant.state = AttachmentUploadGrantState::Cancelled; - None - } - AttachmentUploadGrantState::Cancelled => None, - AttachmentUploadGrantState::Completed(file) => { - let file = file.clone(); - grants.remove(&path.upload_id); - Some(file) - } - } - }; - if let Some(file) = completed { - let worker = resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &file.artifact_id) - .map_err(|error| error.into_error())?; - } - Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) + Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) + }) + .await } async fn scoped_delete_runtime_worker_uploaded_file( @@ -26622,8 +28302,13 @@ async fn post_auth_bootstrap_user( State(api): State, Json(request): Json, ) -> ApiResult> { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - Ok(Json(AuthUserResponse { user })) + api.store + .db_executor() + .run(move || { + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + Ok(Json(AuthUserResponse { user })) + }) + .await } async fn post_passkey_registration_options( @@ -26631,48 +28316,53 @@ async fn post_passkey_registration_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let exclude_credentials = passkeys_for_user(&api, &user.user_id)? - .into_iter() - .map(|passkey| passkey.cred_id().clone()) - .collect(); - let user_unique_id = Uuid::now_v7(); - let (public_key, state) = webauthn - .start_passkey_registration( - user_unique_id, - &user.handle, - &user.display_name, - Some(exclude_credentials), - ) - .map_err(|error| auth_error("webauthn_registration_options_failed", &error.to_string()))?; - let challenge_id = new_id("webauthn-registration"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_registration".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some( - serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?, - ), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyRegistrationOptionsResponse { - challenge_id, - public_key, - })) + api.store + .db_executor() + .run(move || { + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let exclude_credentials = passkeys_for_user(&api, &user.user_id)? + .into_iter() + .map(|passkey| passkey.cred_id().clone()) + .collect(); + let user_unique_id = Uuid::now_v7(); + let (public_key, state) = webauthn + .start_passkey_registration( + user_unique_id, + &user.handle, + &user.display_name, + Some(exclude_credentials), + ) + .map_err(|error| { + auth_error("webauthn_registration_options_failed", &error.to_string()) + })?; + let challenge_id = new_id("webauthn-registration"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_registration".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some(serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyRegistrationOptionsResponse { + challenge_id, + public_key, + })) + }) + .await } fn complete_passkey_registration( @@ -26749,50 +28439,53 @@ async fn post_passkey_login_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - let user = match request.handle.as_deref() { - Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, - None => api.store.any_user()?, - } - .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; - let passkeys = passkeys_for_user(&api, &user.user_id)?; - if passkeys.is_empty() { - return Err(auth_error( - "passkey_not_registered", - "user has no registered passkey credentials", - ) - .into()); - } - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let (public_key, state) = webauthn - .start_passkey_authentication(&passkeys) - .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; - let challenge_id = new_id("webauthn-login"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_login".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some( - serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?, - ), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyLoginOptionsResponse { - challenge_id, - public_key, - })) + api.store + .db_executor() + .run(move || { + let user = match request.handle.as_deref() { + Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, + None => api.store.any_user()?, + } + .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; + let passkeys = passkeys_for_user(&api, &user.user_id)?; + if passkeys.is_empty() { + return Err(auth_error( + "passkey_not_registered", + "user has no registered passkey credentials", + ) + .into()); + } + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let (public_key, state) = webauthn + .start_passkey_authentication(&passkeys) + .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; + let challenge_id = new_id("webauthn-login"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_login".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some(serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyLoginOptionsResponse { + challenge_id, + public_key, + })) + }) + .await } fn complete_passkey_login( @@ -26910,48 +28603,53 @@ async fn post_device_login_start( State(api): State, Json(request): Json, ) -> ApiResult> { - const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; + api.store + .db_executor() + .run(move || { + const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; - let auth = auth_public_config(&api.config); - let verification_uri = format!( - "{}/login/device", - auth.public_base_url.trim_end_matches('/') - ); - for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { - let device_code = mint_secret("yoi_device"); - let user_code = new_user_code(); - let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); - let flow = DeviceLoginFlowRecord { - device_code: device_code.clone(), - user_code: user_code.clone(), - verification_uri: verification_uri.clone(), - client_name: request.client_name.clone(), - user_id: None, - api_token_id: None, - issued_access_token: None, - created_at: crate::auth::now_rfc3339(), - expires_at: rfc3339_after(Duration::minutes(10)), - approved_at: None, - consumed_at: None, - }; - if !api.store.try_create_device_login_flow(&flow)? { - continue; - } - return Ok(Json(DeviceLoginStartResponse { - device_code, - user_code, - verification_uri, - verification_uri_complete, - expires_in: 600, - interval: 5, - })); - } + let auth = auth_public_config(&api.config); + let verification_uri = format!( + "{}/login/device", + auth.public_base_url.trim_end_matches('/') + ); + for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { + let device_code = mint_secret("yoi_device"); + let user_code = new_user_code(); + let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); + let flow = DeviceLoginFlowRecord { + device_code: device_code.clone(), + user_code: user_code.clone(), + verification_uri: verification_uri.clone(), + client_name: request.client_name.clone(), + user_id: None, + api_token_id: None, + issued_access_token: None, + created_at: crate::auth::now_rfc3339(), + expires_at: rfc3339_after(Duration::minutes(10)), + approved_at: None, + consumed_at: None, + }; + if !api.store.try_create_device_login_flow(&flow)? { + continue; + } + return Ok(Json(DeviceLoginStartResponse { + device_code, + user_code, + verification_uri, + verification_uri_complete, + expires_in: 600, + interval: 5, + })); + } - Err(auth_error( - "device_login_code_allocation_failed", - "could not allocate a unique device login user code", - ) - .into()) + Err(auth_error( + "device_login_code_allocation_failed", + "could not allocate a unique device login user code", + ) + .into()) + }) + .await } fn approve_device_login( @@ -27017,41 +28715,48 @@ async fn post_device_login_poll( State(api): State, Json(request): Json, ) -> ApiResult> { - let Some(flow) = api - .store - .get_device_login_flow_by_device_code(&request.device_code)? - else { - return Err(auth_error("unknown_device_login", "device login flow does not exist").into()); - }; - if is_expired(&flow.expires_at) { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Expired, - access_token: None, - token_type: None, - })); - } - if flow.approved_at.is_none() { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Pending, - access_token: None, - token_type: None, - })); - } - let Some(consumed) = api - .store - .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? - else { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Consumed, - access_token: None, - token_type: None, - })); - }; - Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Approved, - access_token: consumed.issued_access_token, - token_type: Some(DeviceAccessTokenType::Bearer), - })) + api.store + .db_executor() + .run(move || { + let Some(flow) = api + .store + .get_device_login_flow_by_device_code(&request.device_code)? + else { + return Err( + auth_error("unknown_device_login", "device login flow does not exist").into(), + ); + }; + if is_expired(&flow.expires_at) { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Expired, + access_token: None, + token_type: None, + })); + } + if flow.approved_at.is_none() { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Pending, + access_token: None, + token_type: None, + })); + } + let Some(consumed) = api + .store + .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? + else { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Consumed, + access_token: None, + token_type: None, + })); + }; + Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Approved, + access_token: consumed.issued_access_token, + token_type: Some(DeviceAccessTokenType::Bearer), + })) + }) + .await } fn logout_browser_session( @@ -27245,7 +28950,7 @@ async fn resolve_actor( headers: &HeaderMap, ) -> ApiResult> { let cookie_name = auth_public_config(&api.config).cookie_name; - Ok(resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await?) + Ok(resolve_request_actor(api.store.clone(), headers, &cookie_name).await?) } async fn require_actor(api: &ServerAuthApi, headers: &HeaderMap) -> ApiResult { @@ -27337,103 +29042,130 @@ async fn list_tickets( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(30).clamp(1, 100); - let states = query - .states - .as_deref() - .map(|states| { - states - .split(',') - .filter(|state| !state.is_empty()) - .map(str::to_string) - .collect::>() + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(30).clamp(1, 100); + let states = query + .states + .as_deref() + .map(|states| { + states + .split(',') + .filter(|state| !state.is_empty()) + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default(); + let crate::records::TicketSummaryPage { + items, + page, + invalid_records, + record_authority, + } = api + .authority + .list_ticket_page(crate::records::TicketListProjectionRequest { + states, + limit: Some(limit), + cursor: query.cursor, + })?; + Ok(Json(crate::records::TicketListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + page, + invalid_records, + record_authority, + })) }) - .unwrap_or_default(); - let crate::records::TicketSummaryPage { - items, - page, - invalid_records, - record_authority, - } = api - .authority - .list_ticket_page(crate::records::TicketListProjectionRequest { - states, - limit: Some(limit), - cursor: query.cursor, - })?; - Ok(Json(crate::records::TicketListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - page, - invalid_records, - record_authority, - })) + .await } async fn get_ticket( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - Ok(Json(api.authority.ticket(&id)?)) + api.store + .db_executor() + .run(move || Ok(Json(api.authority.ticket(&id)?))) + .await } async fn list_objectives( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(api.config.max_records).min(1000); - let ProjectRecordList { - items, - invalid_records, - record_authority, - } = api.authority.list_objectives(limit)?; - Ok(Json(server_api::ObjectiveListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - invalid_records, - record_authority, - })) + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(api.config.max_records).min(1000); + let ProjectRecordList { + items, + invalid_records, + record_authority, + } = api.authority.list_objectives(limit)?; + Ok(Json(server_api::ObjectiveListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + invalid_records, + record_authority, + })) + }) + .await } async fn get_objective( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - Ok(Json(api.authority.objective(&id)?)) + api.store + .db_executor() + .run(move || Ok(Json(api.authority.objective(&id)?))) + .await } async fn list_repositories( State(api): State, ) -> ApiResult> { - let RepositoryListProjection { items, diagnostics } = api.live_repository_reader()?.list(); - Ok(Json(RepositoryListResponse { - workspace_id: api.config.workspace_id, - items, - source: "workspace-control-plane".to_string(), - diagnostics, - })) + api.store + .db_executor() + .run(move || { + let RepositoryListProjection { items, diagnostics } = + api.live_repository_reader()?.list(); + Ok(Json(RepositoryListResponse { + workspace_id: api.config.workspace_id, + items, + source: "workspace-control-plane".to_string(), + diagnostics, + })) + }) + .await } async fn repository_detail( State(api): State, AxumPath(repository_key): AxumPath, ) -> ApiResult> { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let item = repository_lookup( - api.live_repository_reader()? - .summary_by_key(&repository_key), - )?; - Ok(Json(RepositoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - item, - source: "workspace-control-plane".to_string(), - })) + api.store + .db_executor() + .run(move || { + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let item = repository_lookup( + api.live_repository_reader()? + .summary_by_key(&repository_key), + )?; + Ok(Json(RepositoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + item, + source: "workspace-control-plane".to_string(), + })) + }) + .await } async fn repository_log( @@ -27441,63 +29173,81 @@ async fn repository_log( AxumPath(repository_key): AxumPath, Query(query): Query, ) -> ApiResult> { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let RepositoryLogRead { - repository_key, - default_selector, - limit, - commits, - diagnostics, - } = repository_lookup( - api.live_repository_reader()? - .recent_log(&repository_key, query.limit), - )?; - Ok(Json(RepositoryLogResponse { - workspace_id: api.config.workspace_id, - repository_key, - default_selector, - limit, - items: commits, - diagnostics, - })) + api.store + .db_executor() + .run(move || { + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let RepositoryLogRead { + repository_key, + default_selector, + limit, + commits, + diagnostics, + } = repository_lookup( + api.live_repository_reader()? + .recent_log(&repository_key, query.limit), + )?; + Ok(Json(RepositoryLogResponse { + workspace_id: api.config.workspace_id, + repository_key, + default_selector, + limit, + items: commits, + diagnostics, + })) + }) + .await } async fn list_hosts( State(api): State, ) -> ApiResult>> { - let limit = api.config.max_records.min(200); - let runtime_hosts = api.runtime.list_hosts(limit); - Ok(Json(InternalRuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtime_hosts.items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_hosts.diagnostics, - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtime_hosts = api.runtime.list_hosts(limit); + Ok(Json(InternalRuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtime_hosts.items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_hosts.diagnostics, + })) + }) + .await } async fn list_runtimes( State(api): State, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let runtimes = api.runtime.list_runtimes(limit); - Ok(Json(server_api::RuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtimes.items.into_iter().map(Into::into).collect(), - source: "worker_runtime_registry".to_string(), - diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtimes = api.runtime.list_runtimes(limit); + Ok(Json(server_api::RuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtimes.items.into_iter().map(Into::into).collect(), + source: "worker_runtime_registry".to_string(), + diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), + })) + }) + .await } async fn list_workers( State(api): State, ) -> ApiResult> { - workers_response(api).map(Json) + api.store + .db_executor() + .run(move || workers_response(api).map(Json)) + .await } async fn create_remote_runtime( @@ -27669,6 +29419,19 @@ fn runtime_removal_config_guard( async fn execute_runtime_removal( api: &WorkspaceApi, operation: RuntimeRemovalOperation, +) -> ApiResult { + let api = api.clone(); + // Retain registration/commit recovery as one owned lifecycle workflow; + // SQL transaction boundaries and commit-before-Runtime order are unchanged. + run_host_workflow(api.host_executors.host()?, async move { + execute_runtime_removal_on_host(&api, operation).await + }) + .await +} + +async fn execute_runtime_removal_on_host( + api: &WorkspaceApi, + operation: RuntimeRemovalOperation, ) -> ApiResult { if operation.state == RuntimeRemovalOperationState::Succeeded { return Ok(operation); @@ -27871,9 +29634,16 @@ async fn remove_remote_runtime( return Ok(Json(runtime_removal_response(operation))); } + let config_api = api.clone(); let config_state = api - .config_store - .load_workspace_config(&api.config.workspace_id)? + .store + .db_executor() + .run(move || { + config_api + .config_store + .load_workspace_config(&config_api.config.workspace_id) + }) + .await? .ok_or_else(|| { Error::RegistryInconsistency(format!( "Workspace {} has no active configuration", @@ -27954,7 +29724,10 @@ async fn test_runtime_connection( async fn get_worker_launch_options( State(api): State, ) -> ApiResult> { - Ok(Json(worker_launch_options_response(&api)?)) + api.store + .db_executor() + .run(move || Ok(Json(worker_launch_options_response(&api)?))) + .await } fn working_directory_request_from_repository( @@ -28133,6 +29906,27 @@ async fn create_workspace_worker_authenticated( request: CreateWorkspaceWorkerRequest, resolved_control_operation: Option, context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + create_workspace_worker_authenticated_on_host( + api, + headers, + request, + resolved_control_operation, + context.as_ref(), + ) + .await + }) + .await +} + +async fn create_workspace_worker_authenticated_on_host( + api: WorkspaceApi, + headers: HeaderMap, + request: CreateWorkspaceWorkerRequest, + resolved_control_operation: Option, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let claim_lock = request .ticket_assignment @@ -28807,30 +30601,44 @@ fn backend_resource_error_status(error: &BackendResourceError) -> StatusCode { async fn get_companion_status( State(api): State, ) -> ApiResult> { - Ok(Json(api.companion.status())) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.status()))) + .await } async fn get_companion_transcript( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(api.config.max_records).min(200); - let start = query.start.unwrap_or(0); - Ok(Json(api.companion.transcript(start, limit))) + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(api.config.max_records).min(200); + let start = query.start.unwrap_or(0); + Ok(Json(api.companion.transcript(start, limit))) + }) + .await } async fn post_companion_message( State(api): State, Json(request): Json, ) -> ApiResult> { - Ok(Json(api.companion.send_message(request))) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.send_message(request)))) + .await } async fn post_companion_cancel( State(api): State, Json(request): Json, ) -> ApiResult> { - Ok(Json(api.companion.cancel(request))) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.cancel(request)))) + .await } fn runtime_contract_restore( @@ -28929,25 +30737,30 @@ async fn get_runtime_worker( State(api): State, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let worker = api - .runtime - .worker(&worker_ref) - .map_err(|err| err.into_error())?; - let record = sync_worker_observation(&api, &worker)?; - let links = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; - let workdirs = api - .store - .list_workdir_registry(&api.config.workspace_id, 500)?; - let updated_at = record.updated_at.clone(); - let worker = - project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; - Ok(Json(server_api::RuntimeWorkerShowResponse { - worker, - updated_at, - })) + api.host_executors + .host()? + .run(move || { + let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let worker = api + .runtime + .worker(&worker_ref) + .map_err(|err| err.into_error())?; + let record = sync_worker_observation(&api, &worker)?; + let links = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; + let workdirs = api + .store + .list_workdir_registry(&api.config.workspace_id, 500)?; + let updated_at = record.updated_at.clone(); + let worker = + project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; + Ok(Json(server_api::RuntimeWorkerShowResponse { + worker, + updated_at, + })) + }) + .await } async fn restore_runtime_worker_with_context( @@ -28957,7 +30770,7 @@ async fn restore_runtime_worker_with_context( context: WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; let mut result = worker.restore(&context, request).await?; let projected_worker = if let Some(worker) = result.worker.as_ref() { // A replayed terminal receipt is historical evidence, not a new Runtime observation. @@ -29022,33 +30835,38 @@ async fn list_runtime_workers( AxumPath(runtime_id): AxumPath, Query(query): Query, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let (runtime_workers, source) = match query.status { - Some(RuntimeWorkersStatusFilter::Stopped) => ( - api.runtime - .list_stopped_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry_stopped", - ), - None => ( - api.runtime - .list_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry", - ), - }; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::RuntimeWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: source.to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let (runtime_workers, source) = match query.status { + Some(RuntimeWorkersStatusFilter::Stopped) => ( + api.runtime + .list_stopped_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry_stopped", + ), + None => ( + api.runtime + .list_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry", + ), + }; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::RuntimeWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: source.to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) + }) + .await } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -29449,6 +31267,25 @@ async fn create_runtime_worker_authenticated( AxumPath(runtime_id): AxumPath, Json(request): Json, context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + create_runtime_worker_authenticated_on_host( + State(api), + AxumPath(runtime_id), + Json(request), + context.as_ref(), + ) + .await + }) + .await +} + +async fn create_runtime_worker_authenticated_on_host( + State(api): State, + AxumPath(runtime_id): AxumPath, + Json(request): Json, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let mut request = worker_spawn_request_from_api(request)?; let headers = context @@ -29650,17 +31487,22 @@ async fn sync_runtime_config_bundle( AxumPath(runtime_id): AxumPath, Json(request): Json, ) -> ApiResult> { - let bundle = serde_json::from_value::(request.bundle) - .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; - let result = api - .runtime - .sync_config_bundle(&runtime_id, bundle) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) + api.host_executors + .host()? + .run(move || { + let bundle = serde_json::from_value::(request.bundle) + .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; + let result = api + .runtime + .sync_config_bundle(&runtime_id, bundle) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) + }) + .await } async fn check_runtime_config_bundle( @@ -29668,21 +31510,26 @@ async fn check_runtime_config_bundle( AxumPath((runtime_id, bundle_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - let result = api - .runtime - .check_config_bundle( - &runtime_id, - ConfigBundleRef { - id: bundle_id, - digest: query.digest, - }, - ) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) + api.host_executors + .host()? + .run(move || { + let result = api + .runtime + .check_config_bundle( + &runtime_id, + ConfigBundleRef { + id: bundle_id, + digest: query.digest, + }, + ) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) + }) + .await } #[cfg(test)] @@ -29702,7 +31549,7 @@ async fn dispatch_runtime_worker_input( worker_id: String, request: WorkerInputRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; let result = worker .input(&WorkerOperationContext::Backend, request) .await?; @@ -29713,11 +31560,16 @@ async fn delete_runtime_worker_uploaded_file( State(api): State, AxumPath((runtime_id, worker_id, artifact_id)): AxumPath<(String, String, String)>, ) -> ApiResult> { - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &artifact_id) - .map_err(|err| err.into_error())?; - Ok(Json(WorkerFileDeleteResponse { deleted: true })) + api.host_executors + .host()? + .run(move || { + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &artifact_id) + .map_err(|err| err.into_error())?; + Ok(Json(WorkerFileDeleteResponse { deleted: true })) + }) + .await } async fn runtime_worker_completions( @@ -29725,13 +31577,18 @@ async fn runtime_worker_completions( AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, Json(request): Json, ) -> ApiResult> { - let request = worker_completions_request_from_api(request)?; - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let result = api - .runtime - .worker_completions(&worker, request) - .map_err(|err| err.into_error())?; - Ok(Json(worker_completions_result_to_api(result)?)) + api.host_executors + .host()? + .run(move || { + let request = worker_completions_request_from_api(request)?; + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let result = api + .runtime + .worker_completions(&worker, request) + .map_err(|err| err.into_error())?; + Ok(Json(worker_completions_result_to_api(result)?)) + }) + .await } fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource { @@ -29792,7 +31649,7 @@ async fn worker_protocol_ws( headers: HeaderMap, ws: WebSocketUpgrade, ) -> Response { - let worker = match WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) { + let worker = match WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await { Ok(worker) => worker, Err(error) => return ApiError::from(error).into_response(), }; @@ -30300,23 +32157,28 @@ async fn list_host_workers( State(api): State, AxumPath(host_id): AxumPath, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let runtime_workers = api - .runtime - .list_workers_for_host(&host_id, limit) - .map_err(|err| err.into_error())?; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::HostWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtime_workers = api + .runtime + .list_workers_for_host(&host_id, limit) + .map_err(|err| err.into_error())?; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::HostWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) + }) + .await } fn project_workspace_worker_with_attachments( @@ -32875,9 +34737,27 @@ impl From for ApiError { } } +impl From for server_api::RepositoryApiError { + fn from(error: Error) -> Self { + ApiError::from(error).into_repository_api_error() + } +} + impl From for ApiError { fn from(error: Error) -> Self { let diagnostics = match &error { + Error::DbCapacityExceeded + | Error::DbExecutorUnavailable + | Error::DbOperationOutcomeUnknown => vec![RuntimeDiagnostic { + code: match &error { + Error::DbCapacityExceeded => "db_capacity_exceeded", + Error::DbExecutorUnavailable => "db_executor_unavailable", + _ => "db_operation_outcome_unknown", + } + .into(), + severity: HostDiagnosticSeverity::Error, + message: error.to_string(), + }], Error::WorkdirSessionOpenFailed { transport, .. } => vec![RuntimeDiagnostic { code: "workdir_session_open_failed".to_string(), severity: HostDiagnosticSeverity::Error, @@ -33101,6 +34981,8 @@ fn api_error_status(error: &Error) -> StatusCode { { StatusCode::SERVICE_UNAVAILABLE } + Error::DbCapacityExceeded | Error::DbExecutorUnavailable => StatusCode::SERVICE_UNAVAILABLE, + Error::DbOperationOutcomeUnknown => StatusCode::SERVICE_UNAVAILABLE, Error::WorkspaceSigningIdentity { .. } => StatusCode::SERVICE_UNAVAILABLE, Error::WorkdirSessionOpenFailed { .. } | Error::RuntimeOperationFailed { .. } => { StatusCode::BAD_GATEWAY @@ -33158,6 +35040,14 @@ impl IntoResponse for ApiError { } } +#[cfg(test)] +#[path = "server_db_execution_tests.rs"] +mod db_execution_tests; + +#[cfg(test)] +#[path = "server_assignment_db_execution_tests.rs"] +mod assignment_db_execution_tests; + #[cfg(test)] mod tests { mod assignment_delivery_tests { @@ -34101,6 +35991,73 @@ mod tests { ); } + #[tokio::test] + async fn cancelled_workdir_db_waiter_retains_session_lock_until_mutation_finishes() { + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let api = test_api(dir.path()).await; + let worker = RuntimeWorkerRef::new("guard-runtime", "guard-worker"); + let lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(lock.clone().lock_owned().await); + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let record = WorkdirRegistryRecord { + workspace_id: TEST_WORKSPACE_ID.to_string(), + workdir_id: "cancelled-waiter-workdir".to_string(), + display_name: Some("Accepted mutation".to_string()), + source: WorkdirRegistrySource::Repository { + runtime_id: worker.runtime_id.clone(), + repository_id: api.config.repositories[0].id.clone(), + }, + creation_selector: None, + creation_ref: None, + creation_tree: None, + current_selector: None, + current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, + materialization_status: "present".to_string(), + cleanliness: "unknown".to_string(), + created_at: now.clone(), + updated_at: now, + }; + let (started, accepted) = tokio::sync::oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let stage_api = api.clone(); + let waiter = tokio::spawn(async move { + run_worker_workdir_db_stage_locked(&stage_api, &guard, move |api| { + let _ = started.send(()); + released + .recv_timeout(std::time::Duration::from_secs(5)) + .map_err(|error| Error::Store(error.to_string()))?; + api.store.upsert_workdir_registry(&record)?; + Ok::<_, Error>(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), accepted) + .await + .unwrap() + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + // This is the same Worker lock, not an executor-capacity proxy. + let still_locked = lock.try_lock().is_err(); + release.send(()).unwrap(); + assert!( + still_locked, + "accepted mutation lost its Worker session guard" + ); + let _guard = tokio::time::timeout(std::time::Duration::from_secs(5), lock.lock()) + .await + .expect("accepted mutation did not release its Worker lock"); + let persisted = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, "cancelled-waiter-workdir") + .unwrap() + .expect("accepted mutation was cancelled with its waiter"); + assert_eq!(persisted.display_name.as_deref(), Some("Accepted mutation")); + } + #[tokio::test] async fn backend_expiry_fences_offline_grant_and_persists_cleanup_completion() { let dir = tempfile::tempdir().unwrap(); @@ -34294,10 +36251,10 @@ mod tests { inject_external_workdir_cleanup_failure(&grant.workdir_id); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = session_lock.lock().await; + let session_guard = Arc::new(session_lock.lock_owned().await); let error = tokio::time::timeout( std::time::Duration::from_secs(1), - open_current_worker_workdir_session_locked(&api, &worker, &link), + open_current_worker_workdir_session_locked(&api, &worker, &link, &session_guard), ) .await .expect("session-open expiry cleanup must not reacquire its Worker lock") @@ -34842,10 +36799,11 @@ mod tests { ); let session_lock = current_worker_session_lock(&api, &worker_ref); - let session_guard = session_lock.lock().await; - let session = open_current_worker_workdir_session_locked(&api, &worker_ref, link) - .await - .unwrap(); + let session_guard = Arc::new(session_lock.lock_owned().await); + let session = + open_current_worker_workdir_session_locked(&api, &worker_ref, link, &session_guard) + .await + .unwrap(); drop(session_guard); assert_eq!(session.capabilities(), expected_capabilities); if index == 0 { @@ -35268,11 +37226,15 @@ mod tests { workdir::WorkdirSessionCapabilities::ALL ); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = session_lock.lock().await; - let broker_session = - open_current_worker_workdir_session_locked(&api, &worker, &external_link) - .await - .unwrap(); + let session_guard = Arc::new(session_lock.lock_owned().await); + let broker_session = open_current_worker_workdir_session_locked( + &api, + &worker, + &external_link, + &session_guard, + ) + .await + .unwrap(); assert_eq!( broker_session.capabilities(), workdir::WorkdirSessionCapabilities::ALL @@ -36687,7 +38649,7 @@ mod tests { cached_status: "connected".to_owned(), timeout: std::time::Duration::from_secs(1), }); - SqliteWorkspaceStore::open(&api.config.database_path) + SqliteWorkspaceStore::open_for_test(&api.config.database_path) .unwrap() .upsert_workspace_runtime_binding( WorkspaceRuntimeBinding { @@ -38194,9 +40156,9 @@ mod tests { async fn workspace_api_does_not_register_a_missing_workspace() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); - let error = match WorkspaceApi::new(config, store.clone()).await { + let error = match WorkspaceApi::new_for_test(config, store.clone()).await { Ok(_) => panic!("missing Workspace registration must fail closed"), Err(error) => error, }; @@ -38213,11 +40175,12 @@ mod tests { let workspace = tempfile::tempdir().unwrap(); init_clean_git_workspace(workspace.path()); let config = test_server_config(workspace.path()); - let store = Arc::new(SqliteWorkspaceStore::open(config.database_path.clone()).unwrap()); + let store = + Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); - let api = WorkspaceApi::new(config, store).await.unwrap(); + let api = WorkspaceApi::new_for_test(config, store).await.unwrap(); let workspace_id = api.config.workspace_id.clone(); let result = scoped_start_workspace_orchestrator( @@ -38272,7 +40235,7 @@ mod tests { let workspace = tempfile::tempdir().unwrap(); init_clean_git_workspace(workspace.path()); let config = test_server_config(workspace.path()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -42816,8 +44779,8 @@ mod tests { let restored_config = test_server_config(&root) .with_embedded_runtime_store_root(root.join("restored-embedded-runtime")); let restored_store = - SqliteWorkspaceStore::open(restored_config.database_path.clone()).unwrap(); - let restored = WorkspaceApi::new(restored_config, Arc::new(restored_store)) + SqliteWorkspaceStore::open_for_test(restored_config.database_path.clone()).unwrap(); + let restored = WorkspaceApi::new_for_test(restored_config, Arc::new(restored_store)) .await .unwrap(); assert!( @@ -43376,7 +45339,7 @@ mod tests { } } - struct DeterministicExecutionBackend { + pub(super) struct DeterministicExecutionBackend { contexts: std::sync::Mutex< std::collections::HashMap< worker_runtime::identity::WorkerRef, @@ -43445,7 +45408,7 @@ mod tests { .store(true, std::sync::atomic::Ordering::SeqCst); } - fn before_input_returns( + pub(super) fn before_input_returns( &self, hook: impl FnOnce(&worker_runtime::identity::WorkerRef) + Send + 'static, ) { @@ -43485,7 +45448,7 @@ mod tests { execution.publish_event(event).unwrap(); } - fn take_input_request_ids(&self) -> Vec> { + pub(super) fn take_input_request_ids(&self) -> Vec> { std::mem::take( &mut *self .input_request_ids @@ -43494,7 +45457,7 @@ mod tests { ) } - fn take_inputs(&self) -> Vec<(worker_runtime::identity::WorkerRef, String)> { + pub(super) fn take_inputs(&self) -> Vec<(worker_runtime::identity::WorkerRef, String)> { std::mem::take(&mut *self.inputs.lock().expect("inputs lock")) } @@ -44751,6 +46714,17 @@ mod tests { .iter() .all(|(_, content)| { !content.contains("[Ticket item checker advisory]") }) ); + // Terminal SQL state is not actual cleanup completion. Keep this + // fixture's Tokio driver alive through accepted Host/cleanup work before + // dropping it, so a pending teardown timer cannot run on a dead runtime. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + api.host_executors.host().unwrap().drain().await.unwrap(); + api.host_executors.cleanup().unwrap().drain().await.unwrap(); + api.worker_projection.shutdown_async().await; + api.store.db_executor().drain().await.unwrap(); + }) + .await + .expect("checker fixture must finish accepted cleanup before runtime shutdown"); } #[tokio::test] @@ -45143,7 +47117,8 @@ mod tests { notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), }; - let store = Arc::new(SqliteWorkspaceStore::open(config.database_path.clone()).unwrap()); + let store = + Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); @@ -45281,7 +47256,8 @@ mod tests { drop(store); drop(execution); - let restarted_store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let restarted_store = + SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let restarted_execution = Arc::new(DeterministicExecutionBackend::default()); restarted_execution.accept_restores(); let restarted = WorkspaceApi::new_with_execution_backend( @@ -45430,7 +47406,10 @@ mod tests { protocol::WorkerRunState::Running, )) })); - assert_eq!( + // Cleanup admission precedes its durable claim. The accepted job may + // still be queued, but neither queued nor executing ownership may stop + // this Run before the exact tracked finish marker below is published. + assert!(matches!( api.store .get_backend_job_attempt( TEST_WORKSPACE_ID, @@ -45440,8 +47419,8 @@ mod tests { .unwrap() .unwrap() .worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Executing) - ); + Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Executing) + )); execution.publish_worker_state( &worker, @@ -45468,7 +47447,7 @@ mod tests { async fn backend_job_delivery_recovery_replays_same_tracked_notification_id() { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend( config.clone(), @@ -46043,7 +48022,7 @@ mod tests { if let Some(parent) = config.database_path.parent() { fs::create_dir_all(parent).unwrap(); } - SqliteWorkspaceStore::open(&config.database_path).unwrap() + SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap() } #[tokio::test] @@ -46104,7 +48083,7 @@ mod tests { .unwrap(); let mut template = test_server_config(dir.path()); template.static_assets_dir = Some(static_dir); - let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); let app = build_workspace_server_router(template, store) .await .unwrap(); @@ -46187,7 +48166,7 @@ mod tests { async fn repository_free_workspace_can_be_created_selected_and_used_then_register_repository() { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let owner = seed_test_api_token(store.as_ref(), "empty-owner"); let foreign = seed_test_api_token(store.as_ref(), "empty-foreign"); let app = build_workspace_server_router(config.clone(), store.clone()) @@ -46439,7 +48418,7 @@ mod tests { .. } = &config.auth; let expected_origin = expected_origin.clone(); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); store .upsert_account(&AccountRecord { account_id: "account-auth".to_owned(), @@ -47716,7 +49695,7 @@ mod tests { .unwrap(); let mut template = test_server_config(dir.path()); template.static_assets_dir = Some(static_dir); - let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); let token = seed_test_api_token(store.as_ref(), "two-workspaces"); let catalog = WorkspaceCatalogService::new( store.clone(), @@ -47857,7 +49836,7 @@ mod tests { let repository = dir.path().join("repository"); std::fs::create_dir_all(repository.join(".git")).unwrap(); let template = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); let app = build_workspace_server_router(template, store) .await .unwrap(); @@ -47926,7 +49905,7 @@ mod tests { async fn memory_consolidation_backlog_ignores_legacy_filesystem_staging() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -47959,7 +49938,7 @@ mod tests { async fn memory_consolidation_reuses_existing_idle_worker() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -51400,7 +53379,7 @@ mod tests { workspace_root: impl Into, ) -> (WorkspaceApi, Arc) { let config = test_server_config(workspace_root); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) @@ -51424,7 +53403,7 @@ mod tests { config.repositories[0].source = source; config.repositories[0].path = None; config.repositories[0].default_selector = Some("develop".to_string()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -51453,7 +53432,7 @@ mod tests { docs.path = Some(docs_root); docs.default_selector = Some("develop".to_string()); config.repositories.push(docs); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -51888,6 +53867,7 @@ mod tests { store, Arc::new(DeterministicExecutionBackend::default()), BackendResourceBroker::default(), + HostExecutors::default(), Some(dispatcher.clone()), ) .await @@ -52133,7 +54113,7 @@ mod tests { cached_status: "connected".to_string(), timeout: std::time::Duration::from_secs(1), }); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let owner_account_id = format!("account-{TEST_WORKSPACE_ID}"); store .upsert_account(&crate::store::AccountRecord { @@ -52433,7 +54413,8 @@ mod tests { let mut revoked = trust; revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string()); - let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap(); + let authority = + SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(); authority .upsert_workspace_runtime_binding(revoked, true) .unwrap(); @@ -53363,7 +55344,7 @@ mod tests { .unwrap() .digest ); - let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); assert_eq!( changed.digest, restarted @@ -53909,7 +55890,7 @@ mod tests { .unwrap(); assert_eq!(capability_changed.items, reattached.items); assert_ne!(capability_changed.digest, reattached.digest); - let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let (restarted_page, restarted_digest) = restarted .list_worker_workdir_links_page_with_digest( TEST_WORKSPACE_ID, @@ -53928,7 +55909,7 @@ mod tests { .unwrap() .run_to_completion(128, std::time::Duration::from_millis(1), None) .unwrap(); - let restored = SqliteWorkspaceStore::from_connection(snapshot).unwrap(); + let restored = SqliteWorkspaceStore::from_connection_for_test(snapshot).unwrap(); assert_eq!( restored .list_worker_workdir_links_page_with_digest( @@ -53973,7 +55954,7 @@ mod tests { ) .unwrap(); assert_ne!(read_write_digest, capability_changed.digest); - let reader = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let reader = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let barrier = Arc::new(std::sync::Barrier::new(2)); std::thread::scope(|scope| { let worker = &fixture.worker; @@ -55137,6 +57118,8 @@ mod tests { .unwrap(); let worker = orchestrator.worker.unwrap(); let worker = RuntimeWorkerRef::new(worker.runtime_id, worker.worker_id); + let mut stop_events = api.worker_projection.subscribe_ordered(10).unwrap(); + let _ = stop_events.take_snapshot(); api.runtime .stop_worker( &worker, @@ -55146,6 +57129,25 @@ mod tests { }, ) .unwrap(); + // A synchronous Runtime stop no longer implies that its queued SQL + // projection was published. Observe stopped before taking the removal + // baseline, rather than mistaking its Upsert for the atomic removal. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let event = stop_events.recv().await.unwrap(); + if event.changes.iter().any(|change| matches!( + change, + crate::store::WorkerCatalogChange::Upsert(record) + if record.registry.worker == worker + && record.observation.as_ref().is_some_and(|observation| + observation.worker.state == protocol::subscription::SubscriptionWorkerState::Stopped) + )) { + break; + } + } + }) + .await + .expect("stopped Runtime observation must commit before removal baseline"); let worker_root = workspace .path() .join(".test-embedded-runtime-store/workers") @@ -55807,7 +57809,7 @@ mod tests { ) .unwrap(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, other_workspace, @@ -55833,7 +57835,7 @@ mod tests { let identity = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-test").unwrap(); configure_runtime_request_auth(&mut api, &identity, "runtime-test"); - let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let memory_settings = store .get_workspace_memory_settings(TEST_WORKSPACE_ID) .unwrap(); @@ -55868,7 +57870,7 @@ mod tests { let proof = issue(); let source = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, TEST_WORKSPACE_ID, @@ -55886,7 +57888,7 @@ mod tests { .unwrap(); let proof = issue(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, TEST_WORKSPACE_ID, @@ -58033,9 +60035,9 @@ mod tests { std::fs::write(static_dir.join("index.html"), "
Yoi Workspace
").unwrap(); std::fs::write(static_dir.join("assets/app.js"), "console.log('yoi');").unwrap(); - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let mut config = test_server_config(dir.path()); - let sqlite_store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let sqlite_store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); sqlite_store .upsert_workspace(&WorkspaceRecord { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -58070,7 +60072,7 @@ mod tests { "API Ticket", ticket::TicketState::Open, ); - let sqlite_store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let sqlite_store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); sqlite_store .upsert_workspace(&WorkspaceRecord { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -60594,7 +62596,7 @@ mod tests { async fn objective_mutation_endpoints_round_trip_through_workspace_authority() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let owner_account_id = format!("account-{TEST_WORKSPACE_ID}"); store .upsert_account(&crate::store::AccountRecord { diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs index 0fcb072af..33ef01ed4 100644 --- a/crates/workspace-server/src/server/subjektiv_job.rs +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -65,70 +65,85 @@ pub(super) fn require_candidate_grant( /// candidate decision. Once Idle is confirmed, stop cancels any legacy surface /// background work; the new Job regenerates it and does not infer surface success. pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult { + let job_api = api.clone(); + let subject_id = subject_id.to_owned(); let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); - let Some(owner) = api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - else { - return Ok(true); + let (worker, complete) = api + .host_executors + .host()? + .run(move || { + let api = &job_api; + let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); + let Some(owner) = api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + else { + return Ok((None, true)); + }; + let mut summary = api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())?; + summary.singleton_key = Some(owner.key.clone()); + require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; + let stopped = summary.state.eq_ignore_ascii_case("stopped"); + let idle = summary.state.eq_ignore_ascii_case("idle") + && summary + .worker_state + .as_ref() + .is_some_and(|s| s.state == protocol::WorkerState::Idle); + if !stopped && !idle { + return Ok((None, false)); + } + if api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .as_ref() + .map(|o| &o.worker) + != Some(&owner.worker) + { + return Err(Error::RepositoryConflict( + "legacy consolidation owner moved during cutover".into(), + ) + .into()); + } + if !summary.state.eq_ignore_ascii_case("stopped") { + let stopped = api + .runtime + .stop_worker( + &owner.worker, + WorkerLifecycleRequest { + reason: Some( + "Retire legacy subjektiv consolidator after foreground completion" + .into(), + ), + ticket_assignment: None, + }, + ) + .map_err(|e| e.into_error())?; + if stopped.state != InternalWorkerOperationState::Accepted { + return Ok((None, false)); + } + } + if !api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())? + .state + .eq_ignore_ascii_case("stopped") + { + return Ok((None, false)); + } + Ok::<_, ApiError>((Some(owner.worker), false)) + }) + .await?; + let Some(worker) = worker else { + return Ok(complete); }; - let mut summary = api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())?; - summary.singleton_key = Some(owner.key.clone()); - require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; - let stopped = summary.state.eq_ignore_ascii_case("stopped"); - let idle = summary.state.eq_ignore_ascii_case("idle") - && summary - .worker_state - .as_ref() - .is_some_and(|s| s.state == protocol::WorkerState::Idle); - if !stopped && !idle { - return Ok(false); - } - if api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - .as_ref() - .map(|o| &o.worker) - != Some(&owner.worker) - { - return Err(Error::RepositoryConflict( - "legacy consolidation owner moved during cutover".into(), - ) - .into()); - } - if !summary.state.eq_ignore_ascii_case("stopped") { - let stopped = api - .runtime - .stop_worker( - &owner.worker, - WorkerLifecycleRequest { - reason: Some( - "Retire legacy subjektiv consolidator after foreground completion".into(), - ), - ticket_assignment: None, - }, - ) - .map_err(|e| e.into_error())?; - if stopped.state != InternalWorkerOperationState::Accepted { - return Ok(false); - } - } - if !api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())? - .state - .eq_ignore_ascii_case("stopped") - { - return Ok(false); - } let removal = WorkerRemovalService::new(api) .execute_target_removal( api.runtime.as_ref(), - &owner.worker, + &worker, "subjektiv consolidation Job cutover", None, ) @@ -141,10 +156,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) - )) .into()); } - Ok(api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - .is_none()) + let store = api.store.clone(); + let workspace_id = api.workspace_id().to_owned(); + api.store + .db_executor() + .run(move || { + store + .current_worker_singleton_owner(&workspace_id, &key) + .map(|owner| owner.is_none()) + .map_err(ApiError::from) + }) + .await } fn bounded_candidate_batch( diff --git a/crates/workspace-server/src/server/tests/api_logging_tests.rs b/crates/workspace-server/src/server/tests/api_logging_tests.rs index e4095b829..2bac1950b 100644 --- a/crates/workspace-server/src/server/tests/api_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/api_logging_tests.rs @@ -806,7 +806,19 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc .unwrap(); }; let ((status, body, events), ()) = tokio::join!( - request_with_logs(build_router(fixture.api.clone()), request), + async { + let result = request_with_logs(build_router(fixture.api.clone()), request).await; + // A pre-dispatch rejection leaves the fake provider waiting on an + // open channel. Fail with the actual response here instead of + // hanging join! until the outer suite timeout hides that cause. + assert_eq!( + result.0, + StatusCode::BAD_GATEWAY, + "early command-start response: {}", + result.1 + ); + result + }, provider ); assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); @@ -832,3 +844,78 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc "post-dispatch failure was retried" ); } + +#[tokio::test] +async fn saturated_host_before_command_start_returns_availability_without_a_provider_operation() { + let mut fixture = manual_worker_assignment_fixture().await; + let identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) + .unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let (sender, mut commands) = tokio::sync::mpsc::channel(1); + let connection = Arc::new(ExternalProviderConnection { + grant_id: "overload-grant".into(), + workdir_id: fixture.main_workdir_id.clone(), + provider_instance_id: "overload-provider".into(), + generation: 1, + expires_at: None, + capabilities: workdir::WorkdirSessionCapabilities::ALL, + admission: Arc::new(tokio::sync::Semaphore::new(1)), + shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + sender, + }); + fixture.runtime.workdir_session_factory = Some(Arc::new(move |_| { + Ok(Arc::new(ExternalProviderWorkdirSession::new( + connection.clone(), + None, + )) as WorkdirSessionHandle) + })); + fixture + .api + .runtime + .register_or_replace(fixture.runtime.clone()); + let host = fixture.api.host_executors.host().unwrap(); + host.drain().await.unwrap(); + let mut releases = Vec::new(); // dropping senders also releases jobs on panic + for _ in 0..2 { + let (release, permit) = std::sync::mpsc::channel::<()>(); + releases.push(release); + let (started, accepted) = tokio::sync::oneshot::channel(); + host.dispatch(move || { + let _ = started.send(()); + let _ = permit.recv_timeout(std::time::Duration::from_secs(10)); + }) + .unwrap(); + accepted.await.unwrap(); + } + for _ in 0..16 { + host.dispatch(|| {}).unwrap(); + } + let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations"); + let request = runtime_source_request( + &identity, + Some(&fixture.worker.worker_id), + "POST", + &path, + serde_json::to_vec(&json!({ + "target_workdir": "checkout", + "operation": {"operation": "command_start", "request": { + "command": "not-dispatched", "timeout_secs": 1, "output_limit": 1024, + "cwd": "", "spill_dir": null, "tool_call_id": "not-dispatched" + }} + })) + .unwrap(), + ); + let (status, body, _) = request_with_logs(build_router(fixture.api.clone()), request).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert!(body.to_string().contains("db_capacity_exceeded"), "{body}"); + // The response is already final but the live fake-provider receiver remains + // Pending. An unbounded join of those two futures hid this causal rejection + // behind the original 600-second suite timeout. No process or command ran. + let provider_command = commands.recv(); + tokio::pin!(provider_command); + assert!(futures::poll!(provider_command.as_mut()).is_pending()); + drop(releases); + host.drain().await.unwrap(); + fixture.api.store.db_executor().drain().await.unwrap(); +} diff --git a/crates/workspace-server/src/server/tests/auth_logging_tests.rs b/crates/workspace-server/src/server/tests/auth_logging_tests.rs index b4b51d7bc..d5bf3334a 100644 --- a/crates/workspace-server/src/server/tests/auth_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/auth_logging_tests.rs @@ -54,7 +54,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_ configure_runtime_request_auth(&mut api, &identity, "runtime-test"); let revoked = RuntimeIdentityMaterial::generate("revoked-runtime-secret").unwrap(); configure_runtime_request_auth(&mut api, &revoked, "revoked-runtime-secret"); - SqliteWorkspaceStore::open(&api.config.database_path) + SqliteWorkspaceStore::open_for_test(&api.config.database_path) .unwrap() .revoke_workspace_runtime_binding( TEST_WORKSPACE_ID, @@ -70,7 +70,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_ let body = "body-secret"; let apps = [ build_router(api.clone()), - workspace_server_router(WorkspaceServerApi::new( + workspace_server_router(WorkspaceServerApi::new_for_test( api.config.clone(), api.store.clone(), )), @@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m &error, ) }); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert!( to_bytes(response.into_body(), usize::MAX) .await @@ -347,7 +347,7 @@ async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() { let path = format!("/api/w/{TEST_WORKSPACE_ID}/repositories"); for app in [ build_router(api.clone()), - workspace_server_router(WorkspaceServerApi::new( + workspace_server_router(WorkspaceServerApi::new_for_test( api.config.clone(), api.store.clone(), )), diff --git a/crates/workspace-server/src/server/tests/drive_tests.rs b/crates/workspace-server/src/server/tests/drive_tests.rs index 1d05ff7db..3b7ec65ca 100644 --- a/crates/workspace-server/src/server/tests/drive_tests.rs +++ b/crates/workspace-server/src/server/tests/drive_tests.rs @@ -40,7 +40,7 @@ async fn cached_drive_handle_rejects_mutation_after_deletion_reservation() { }) .await .unwrap(); - let api = WorkspaceApi::new(config.clone(), store.clone()) + let api = WorkspaceApi::new_for_test(config.clone(), store.clone()) .await .unwrap(); let cached = api.drive().await.unwrap(); @@ -104,7 +104,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature // Use a separate embedded-runtime namespace for the first API: this test // reconstructs Drive authority in-process rather than spawning/restarting // product processes (runtime observers may retain their namespace). - let first_api = WorkspaceApi::new(config.clone(), store.clone()) + let first_api = WorkspaceApi::new_for_test(config.clone(), store.clone()) .await .unwrap(); // Merely constructing a Workspace API must not create blob storage. @@ -160,7 +160,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature .unwrap() .unwrap(); drop(first_api); - let recovered = WorkspaceServerApi::new(config.clone(), store.clone()); + let recovered = WorkspaceServerApi::new_for_test(config.clone(), store.clone()); let api = recovered .api_for_workspace(&config.workspace_id) .await diff --git a/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs b/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs index 02ab40e35..71efdb137 100644 --- a/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs +++ b/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs @@ -39,10 +39,10 @@ impl RealDriveFixture { let temp = tempfile::tempdir().unwrap(); let mut config = test_server_config(temp.path()); config.repositories.clear(); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let owner = seed_test_api_token(store.as_ref(), "drive-web-owner"); let second = seed_test_api_token(store.as_ref(), "drive-web-second"); - let server_api = WorkspaceServerApi::new(config, store.clone()); + let server_api = WorkspaceServerApi::new_for_test(config, store.clone()); let server = LoopbackServer::start(workspace_server_router(server_api.clone())).await; // Exercise T-717's real repositoryless Workspace bootstrap, not a // pre-existing test Workspace with its repository merely hidden in UI. diff --git a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs index 7a4066b39..358b9a7bb 100644 --- a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs +++ b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs @@ -767,6 +767,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) { std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 }) ); assert!(execution.protocol_methods().is_empty()); + // SQLite jobs run on external threads; paused Tokio time cannot observe their + // progress and may auto-advance a reconnect timeout before SQL completes. + tokio::time::resume(); assert_same_worker_reconnect_and_stop(&worker, &context).await; } @@ -844,7 +847,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub } } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress() { let workspace = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs index b54c4bdad..37519b22c 100644 --- a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs +++ b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs @@ -27,7 +27,7 @@ async fn subject_session_callback_completes_while_create_and_deletion_are_waitin ) .unwrap(); let worker_id = reserved.worker_id.to_string(); - let server = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); + let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); // Reuse the fixture API, but exercise the real outer dispatcher and both auth layers. server .apis diff --git a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs index 64ca15dc3..733df3809 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs @@ -144,7 +144,7 @@ impl LaunchFixture { async fn new(remote: bool) -> Self { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let backend = Arc::new(LaunchExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), backend.clone()) diff --git a/crates/workspace-server/src/server/tests/worker_drive_tests.rs b/crates/workspace-server/src/server/tests/worker_drive_tests.rs index 2b24e45a5..481c309ad 100644 --- a/crates/workspace-server/src/server/tests/worker_drive_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_drive_tests.rs @@ -151,7 +151,7 @@ impl Fixture { let temp = tempfile::tempdir().unwrap(); let mut config = test_server_config(temp.path()); config.repositories.clear(); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let mut api = WorkspaceApi::new_with_execution_backend( config, store, diff --git a/crates/workspace-server/src/server/tests/worker_operations_tests.rs b/crates/workspace-server/src/server/tests/worker_operations_tests.rs index 2229bdb59..574c48b8e 100644 --- a/crates/workspace-server/src/server/tests/worker_operations_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_operations_tests.rs @@ -830,7 +830,11 @@ async fn grant_replacement_while_admission_waits_cannot_use_another_grants_lock( }; let admission = handle.set_pinned(&context, true); tokio::pin!(admission); - // Poll deterministically to the held lock before replacing grant authority. + // The first poll can now be waiting for the offloaded authority read, not + // the held grant lock. Drain this fixture's actual SQL, then poll its saved + // reply to reach admission before replacing authority. + assert!(futures::poll!(admission.as_mut()).is_pending()); + api.store.db_executor().drain().await.unwrap(); assert!(futures::poll!(admission.as_mut()).is_pending()); api.store .revoke_worker_control_grant( @@ -1329,8 +1333,9 @@ async fn restore_guard_internal_stale_conflict_retires_intent_for_later_explicit ); assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); // Retirement, unlike in-memory refresh, must survive caller journal restart. - api.config_store = - Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); + api.config_store = Arc::new( + SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(), + ); let fresh = api.runtime.worker(&identity).unwrap(); let next = api .config_store @@ -1490,7 +1495,7 @@ async fn restore_guard_internal_intent_survives_client_store_restart_without_rep ); // Reopen the durable caller journal, discarding all client-side memory of request identity. api.config_store = - Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); + Arc::new(SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap()); let pinned = api .config_store .pin_internal_worker_restore_intent( @@ -1599,7 +1604,7 @@ async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_w crate::repository_source::repository_source_fingerprint(&source); config.repositories[0].source = source; config.repositories[0].path = None; - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); execution.accept_restores(); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs index 9bfbd3364..142ae143e 100644 --- a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs @@ -56,7 +56,7 @@ impl Fixture { api.runtime.register_or_replace(runtime); let now = Arc::new(AtomicI64::new(START)); let clock = now.clone(); - let mut server = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); + let mut server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); server.proof_clock = Arc::new(move || clock.load(Ordering::SeqCst)); server .apis @@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin let f = Fixture::new().await; let mut a = f.attach("1"); let (first, reply) = f.start_read("1", &mut a).await; - let mut second = Box::pin(f.app.clone().oneshot(f.operation("1"))); - // Poll through dispatch/auth until the handler waits on its resource lock. - assert!(futures::poll!(&mut second).is_pending()); + let request = f.operation("1"); + let proof = request + .headers() + .get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER) + .unwrap() + .to_str() + .unwrap(); + let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof) + .unwrap() + .jti; + let mut second = Box::pin(f.app.clone().oneshot(request)); + // DB auth is now genuinely asynchronous. Observe durable proof consumption, + // not a guessed number of polls, before advancing the injected proof clock. + let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap(); + tokio::time::timeout(BOUND, async { + loop { + assert!(futures::poll!(&mut second).is_pending()); + let consumed: bool = connection + .query_row( + "SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)", + [&jti], + |row| row.get(0), + ) + .unwrap(); + if consumed { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("second request never authenticated before resource wait"); assert!( a.try_recv().is_err(), "same session read escaped resource exclusion" @@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin assert_json(first.await.unwrap(), StatusCode::OK).await; assert!(futures::poll!(&mut second).is_pending()); let Some(ExternalProviderCommand::Operation { response, .. }) = - tokio::time::timeout(BOUND, a.recv()).await.unwrap() + tokio::time::timeout(BOUND, async { + tokio::select! { + command = a.recv() => command, + result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()), + } + }).await.unwrap() else { panic!("expected second read") }; @@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits( }), ); let mut deletion = Box::pin(f.app.clone().oneshot(request)); - assert!(futures::poll!(&mut deletion).is_pending()); let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await; - assert!( - gate.admit(false).is_none(), - "deletion start did not fence admission" - ); + tokio::time::timeout(BOUND, async { + loop { + assert!(futures::poll!(&mut deletion).is_pending()); + if gate.admit(false).is_none() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("deletion start did not fence admission after async authentication"); let session_id = session_store::new_session_id().to_string(); let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions"); let callback = f.request( diff --git a/crates/workspace-server/src/server/worker_operations.rs b/crates/workspace-server/src/server/worker_operations.rs index 35b60148f..447ae7e29 100644 --- a/crates/workspace-server/src/server/worker_operations.rs +++ b/crates/workspace-server/src/server/worker_operations.rs @@ -57,6 +57,19 @@ impl WorkerOperationContext { } Ok(()) } + pub(super) async fn from_request_async( + api: &WorkspaceApi, + context: &server_api::ServerRequestContext, + ) -> std::result::Result { + let owned_api = api.clone(); + let context = context.clone(); + api.store + .db_executor() + .run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context))) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? + } + pub(super) fn from_request( api: &WorkspaceApi, context: &server_api::ServerRequestContext, @@ -162,8 +175,20 @@ impl WorkspaceWorker { source: crate::worker_source::VerifiedWorkerMutationSource, reason: &str, ) -> std::result::Result { - self.current_record() - .map_err(|error| error.error.to_string())?; + let removal = self.removal.clone(); + let identity = self.identity.clone(); + self.removal + .store + .db_executor() + .run(move || { + removal + .store + .get_worker_registry(&removal.workspace_id, &identity)? + .ok_or_else(|| Error::UnknownWorker { worker: identity }) + .map(drop) + }) + .await + .map_err(|error| error.to_string())?; self.removal .execute_async( source, @@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor } impl WorkspaceWorker { + pub(crate) async fn resolve_async( + api: &WorkspaceApi, + runtime_id: &str, + reference: &str, + ) -> Result { + let owned_api = api.clone(); + let runtime_id = runtime_id.to_owned(); + let reference = reference.to_owned(); + api.store + .db_executor() + .run(move || Self::resolve(&owned_api, &runtime_id, &reference)) + .await + } + pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result { Self::resolve_with_services( api.clone(), @@ -239,7 +278,36 @@ impl WorkspaceWorker { context: &WorkerOperationContext, permission: &str, ) -> ApiResult>> { - self.current_record()?; + let worker = self.clone(); + let initial_context = context.clone(); + let permission = permission.to_owned(); + let initial_permission = permission.clone(); + let grant = self + .api + .store + .db_executor() + .run(move || -> ApiResult> { + worker.current_record()?; + let WorkerOperationContext::WorkerControl { controller } = initial_context else { + return Ok(None); + }; + if worker + .api + .store + .get_worker_registry(worker.api.workspace_id(), &controller)? + .is_none() + { + return Err(Error::UnknownWorker { worker: controller }.into()); + } + Ok(Some(authorize_known_worker_permission( + &worker.api, + worker.api.workspace_id(), + &controller, + &worker.identity, + &initial_permission, + )?)) + }) + .await?; match context { #[cfg(test)] WorkerOperationContext::Backend => Ok(None), @@ -248,42 +316,35 @@ impl WorkspaceWorker { Ok(None) } WorkerOperationContext::WorkerControl { controller } => { - if self - .api - .store - .get_worker_registry(self.api.workspace_id(), controller)? - .is_none() - { - return Err(Error::UnknownWorker { - worker: controller.clone(), - } - .into()); - } - let grant = authorize_known_worker_permission( - &self.api, - self.api.workspace_id(), - controller, - &self.identity, - permission, - )?; + let grant = grant.expect("Worker control context resolved a grant"); + // Await admission outside the DB lane; the recheck owns the guard + // until it finishes, even if its async waiter is cancelled. let guard = worker_control_lock(&self.api, &grant.grant_id) .lock_owned() .await; - self.current_record()?; - let current_grant = authorize_known_worker_permission( - &self.api, - self.api.workspace_id(), - controller, - &self.identity, - permission, - )?; - if current_grant.grant_id != grant.grant_id { - return Err(Error::RepositoryConflict( - "Worker control grant changed while waiting for admission".into(), - ) - .into()); - } - Ok(Some(guard)) + let worker = self.clone(); + let controller = controller.clone(); + self.api + .store + .db_executor() + .run(move || -> ApiResult<_> { + worker.current_record()?; + let current_grant = authorize_known_worker_permission( + &worker.api, + worker.api.workspace_id(), + &controller, + &worker.identity, + &permission, + )?; + if current_grant.grant_id != grant.grant_id { + return Err(Error::RepositoryConflict( + "Worker control grant changed while waiting for admission".into(), + ) + .into()); + } + Ok(Some(guard)) + }) + .await } } } @@ -293,8 +354,16 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "restore").await?; - self.restore_internal(request) + let guard = self.authorize_operation(context, "restore").await?; + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || { + let _guard = guard; + worker.restore_internal(request) + }) + .await } /// Trusted Backend callers still supply an explicit observed intent. Runtime @@ -312,27 +381,48 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "stop").await?; - let result = self + let guard = self.authorize_operation(context, "stop").await?; + let worker = self.clone(); + let (result, guard) = self .api - .runtime - .stop_worker(&self.identity, request) - .map_err(|e| e.into_error())?; + .host_executors + .host()? + .run(move || -> ApiResult<_> { + let result = worker + .api + .runtime + .stop_worker(&worker.identity, request) + .map_err(|e| e.into_error())?; + Ok((result, guard)) + }) + .await?; if result.state != InternalWorkerOperationState::Accepted { return Ok(result); } parse_runtime_worker_id_for_registry(&self.identity.worker_id)?; let session_lock = current_worker_session_lock(&self.api, &self.identity); - let _session_guard = session_lock.lock().await; + let session_guard = session_lock.lock_owned().await; close_current_worker_session_locked(&self.api, &self.identity).await?; - if let Some(record) = self - .api - .store - .get_worker_registry(self.api.workspace_id(), &self.identity)? - { - sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?; - } - Ok(result) + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || -> ApiResult<_> { + let (_control, _session) = (guard, session_guard); + if let Some(record) = worker + .api + .store + .get_worker_registry(worker.api.workspace_id(), &worker.identity)? + { + sync_linked_workdir_after_worker_stop( + &worker.api, + &worker.identity.runtime_id, + &record, + )?; + } + Ok(result) + }) + .await } pub(super) async fn cancel( @@ -340,12 +430,20 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "cancel").await?; - Ok(self - .api - .runtime - .cancel_worker(&self.identity, request) - .map_err(|e| e.into_error())?) + let guard = self.authorize_operation(context, "cancel").await?; + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || -> ApiResult<_> { + let _guard = guard; + Ok(worker + .api + .runtime + .cancel_worker(&worker.identity, request) + .map_err(|e| e.into_error())?) + }) + .await } pub(super) async fn input( @@ -358,46 +456,61 @@ impl WorkspaceWorker { } else { "send_input" }; - let _guard = self.authorize_operation(context, permission).await?; + let guard = self.authorize_operation(context, permission).await?; let session_lock = current_worker_session_lock(&self.api, &self.identity); - let _session_guard = session_lock.lock().await; - self.ensure_interactive()?; + let session_guard = session_lock.lock_owned().await; + self.ensure_interactive().await?; if matches!(context, WorkerOperationContext::WorkerControl { .. }) && request.kind == WorkerInputKind::User { request.kind = WorkerInputKind::UserIfIdle; } - Ok(self - .api - .runtime - .send_input(&self.identity, request) - .map_err(|e| e.into_error())?) + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || -> ApiResult<_> { + let (_control, _session) = (guard, session_guard); + Ok(worker + .api + .runtime + .send_input(&worker.identity, request) + .map_err(|e| e.into_error())?) + }) + .await } - fn ensure_interactive(&self) -> ApiResult<()> { - if let Some(claim) = self - .api + async fn ensure_interactive(&self) -> ApiResult<()> { + let worker = self.clone(); + self.api .store - .get_pending_ticket_assignment_operation_for_worker( - self.api.workspace_id(), - &self.identity, - )? - { - return Err(Error::TicketAssignmentConflict(format!( - "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", - claim.operation_id, - )).into()); - } - if self - .api - .store - .worker_registry_projection(self.api.workspace_id(), &self.identity)? - .and_then(|projection| projection.job) - .is_some() - { - return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); - } - Ok(()) + .db_executor() + .run(move || -> ApiResult<()> { + if let Some(claim) = worker + .api + .store + .get_pending_ticket_assignment_operation_for_worker( + worker.api.workspace_id(), + &worker.identity, + )? + { + return Err(Error::TicketAssignmentConflict(format!( + "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", + claim.operation_id, + )).into()); + } + if worker + .api + .store + .worker_registry_projection(worker.api.workspace_id(), &worker.identity)? + .and_then(|projection| projection.job) + .is_some() + { + return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); + } + Ok(()) + }) + .await } pub(super) async fn set_pinned( @@ -405,33 +518,44 @@ impl WorkspaceWorker { context: &WorkerOperationContext, pinned: bool, ) -> ApiResult { - let _guard = self.authorize_operation(context, "pin").await?; - let retention_state = if pinned { "pinned" } else { "normal" }; - let changed = self.api.worker_projection.publish_ordered(|| { - self.api - .store - .update_worker_retention( - self.api.workspace_id(), - &self.identity, - retention_state, - now_registry_timestamp().as_str(), - ) - .map(|commit| (!commit.changes.is_empty(), commit)) - })?; - if !changed { - return Err(cleanup_api_error( - &self.identity.runtime_id, - "workspace_worker_retention_unknown_worker", - "Worker is not known to the Backend registry", - )); - } - Ok(WorkerRetentionResponse { - workspace_id: self.api.workspace_id().to_owned(), - runtime_id: self.identity.runtime_id.clone(), - worker_id: self.identity.worker_id.clone(), - pinned, - retention_state: retention_state.into(), - }) + let guard = self.authorize_operation(context, "pin").await?; + let worker = self.clone(); + self.api + .store + .db_executor() + .run(move || -> ApiResult { + // Publication and its DB mutation are one owned stage. Accepted + // work retains control admission through commit and publication. + let _guard = guard; + let retention_state = if pinned { "pinned" } else { "normal" }; + let changed = worker.api.worker_projection.publish_ordered(|| { + worker + .api + .store + .update_worker_retention( + worker.api.workspace_id(), + &worker.identity, + retention_state, + now_registry_timestamp().as_str(), + ) + .map(|commit| (!commit.changes.is_empty(), commit)) + })?; + if !changed { + return Err(cleanup_api_error( + &worker.identity.runtime_id, + "workspace_worker_retention_unknown_worker", + "Worker is not known to the Backend registry", + )); + } + Ok(WorkerRetentionResponse { + workspace_id: worker.api.workspace_id().to_owned(), + runtime_id: worker.identity.runtime_id.clone(), + worker_id: worker.identity.worker_id.clone(), + pinned, + retention_state: retention_state.into(), + }) + }) + .await } /// The caller owns the full cleanup-plan digest check. The removal service @@ -611,7 +735,7 @@ impl WorkspaceWorkerMethodSender { .await?; let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity); let _session_guard = session_lock.lock().await; - self.worker.ensure_interactive()?; + self.worker.ensure_interactive().await?; let WorkerOperationContext::Browser { source, .. } = context else { return Err(Error::WorkspacePermissionDenied( "Browser Worker protocol context required".into(), diff --git a/crates/workspace-server/src/server/workspace_admission.rs b/crates/workspace-server/src/server/workspace_admission.rs index 06c94b9a8..5f8335629 100644 --- a/crates/workspace-server/src/server/workspace_admission.rs +++ b/crates/workspace-server/src/server/workspace_admission.rs @@ -72,6 +72,28 @@ pub(super) struct RequestLease { callback: bool, } +impl crate::db_executor::DbOperationLifetime for RequestLease { + fn retain(&self) -> Box { + // This is continuation of an already admitted request, not fresh admission. + // Retain its drain responsibility before transferring work to the DB owner; + // dropping/aborting the HTTP waiter must not allow deletion past live SQL. + let mut state = self + .gate + .state + .lock() + .expect("Workspace admission poisoned"); + if self.callback { + state.callbacks += 1; + } else { + state.requests += 1; + } + Box::new(RequestLease { + gate: self.gate.clone(), + callback: self.callback, + }) + } +} + impl Drop for RequestLease { fn drop(&mut self) { let mut state = self diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index 7f8c57d56..ee20767bc 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -597,7 +597,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) { let dir = tempfile::tempdir().unwrap(); init_clean_git_workspace(dir.path()); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); diff --git a/crates/workspace-server/src/server_assignment_db_execution_tests.rs b/crates/workspace-server/src/server_assignment_db_execution_tests.rs new file mode 100644 index 000000000..724066895 --- /dev/null +++ b/crates/workspace-server/src/server_assignment_db_execution_tests.rs @@ -0,0 +1,1408 @@ +//! Real SQLite and generated HTTP boundaries for continuing assignment input. +//! Every fixture owns its finite DB/Host lanes and all observer userdata. +use super::tests::DeterministicExecutionBackend; +use super::*; +use crate::SqliteWorkspaceStore; +use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime}; +use rusqlite::{Connection, ffi}; +use std::{ + ffi::c_void, + future::Future, + num::NonZeroUsize, + sync::{ + atomic::{AtomicBool, AtomicUsize}, + mpsc, + }, + thread::JoinHandle, + time::Duration as StdDuration, +}; +use tokio::sync::oneshot; +use worker_runtime::auth::RuntimeIdentityMaterial; + +// The cfg(test) direct Orchestrator adapter seeds its binding for this existing +// fixture Workspace identity. Databases and executors remain per-fixture owners. +const WORKSPACE: &str = "0192f0e8-4d84-7d6e-a000-000000000001"; +const AT: &str = "2026-01-01T00:00:00Z"; +const WATCHDOG: StdDuration = StdDuration::from_secs(10); +const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(5); + +async fn observed(future: impl Future) -> T { + tokio::time::timeout(OBSERVE_LIMIT, future) + .await + .expect("assignment request did not reach the expected SQLite/publication boundary") +} +fn lane(workers: usize, queued: usize) -> DbExecutor { + DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(workers).unwrap(), + queued: NonZeroUsize::new(queued).unwrap(), + }) + .unwrap() +} + +struct Fixture { + api: WorkspaceApi, + store: Arc, + executor: DbExecutor, + execution: Arc, + ticket: ticket::TicketRef, + _dir: tempfile::TempDir, +} +impl Fixture { + async fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let executor = lane(1, 32); + let config = ServerConfig { + workspace_id: WORKSPACE.into(), + workspace_display_name: "Assignment DB fixture".into(), + workspace_created_at: AT.into(), + workspace_execution_root: dir.path().into(), + database_path: dir.path().join("server.db"), + data_root: dir.path().join("data"), + embedded_runtime_store_root: dir.path().join("runtime"), + static_assets_dir: None, + auth: AuthConfig::Passkey { + rp_id: "localhost".into(), + origin: "http://localhost:5173".into(), + public_base_url: "http://localhost:5173".into(), + cookie_name: "assignment_db_session".into(), + }, + max_records: 100, + repositories: Vec::new(), + runtime_event_sources: Vec::new(), + remote_runtime_sources: Vec::new(), + backend_base_url: Some("assignment-db-server".into()), + }; + let store = Arc::new( + SqliteWorkspaceStore::open_for_test(&config.database_path) + .unwrap() + .with_db_executor(executor.clone()), + ); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + store + .upsert_user(&UserRecord { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + created_at: AT.into(), + updated_at: AT.into(), + }) + .unwrap(); + store + .create_browser_session(&crate::store::BrowserSessionRecord { + session_id: "fixture-session".into(), + token_hash: crate::auth::token_hash("fixture-browser-token"), + user_id: "fixture-user".into(), + created_at: AT.into(), + expires_at: "2999-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + store + .create_api_token(&ApiTokenRecord { + token_hash: crate::auth::token_hash("fixture-api-token"), + token_id: "fixture-token".into(), + user_id: "fixture-user".into(), + label: "assignment fixture".into(), + created_at: AT.into(), + expires_at: None, + last_used_at: None, + revoked_at: None, + }) + .unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + // Explicit, fixture-owned finite lanes; never the production singleton. + let host_executors = HostExecutors::default(); + host_executors + .host + .set(Some(lane(2, 16))) + .unwrap_or_else(|_| panic!("Host owner initialized twice")); + host_executors + .cleanup + .set(Some(lane(2, 16))) + .unwrap_or_else(|_| panic!("cleanup owner initialized twice")); + let api = WorkspaceApi::new_with_execution_backend_and_broker( + config, + store.clone(), + execution.clone(), + BackendResourceBroker::default(), + host_executors, + None, + ) + .await + .unwrap(); + let mut input = ticket::NewTicket::new("Continuing assignment input"); + input.workflow_state = Some(TicketState::Open); + let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); + store + .set_current_ticket_role_assignment( + &TicketRoleAssignmentRecord { + workspace_id: WORKSPACE.into(), + ticket_id: ticket.id.clone(), + assignment_id: "fixture-assignment".into(), + role: TicketAssignmentRole::Orchestrator, + principal: TicketAssignmentPrincipal::WorkspaceAgent { + agent_key: "workspace-orchestrator".into(), + }, + assigned_by: "fixture-user".into(), + assigned_at: AT.into(), + }, + None, + "fixture-assign-event", + "fixture-assign-operation", + false, + ) + .unwrap(); + executor.drain().await.unwrap(); + Self { + api, + store, + executor, + execution, + ticket, + _dir: dir, + } + } + fn path(&self, browser: bool, suffix: &str) -> String { + let prefix = if browser { "api/w" } else { "workspaces" }; + // Exercise reference resolution, not merely an internal row ID. + format!( + "/{prefix}/{WORKSPACE}/tickets/{}/{suffix}", + self.ticket.resource_key.as_deref().unwrap() + ) + } + fn request(method: &str, path: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header("cookie", "assignment_db_session=fixture-browser-token") + .header(ORIGIN, "http://localhost:5173") + .body(axum::body::Body::from(body.to_string())) + .unwrap() + } + fn account_request(method: &str, path: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header("authorization", "Bearer fixture-api-token") + .body(axum::body::Body::from(body.to_string())) + .unwrap() + } + fn actor() -> RequestActor { + RequestActor { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + auth_method: ActorAuthMethod::BrowserSession, + } + } + // Contention tests isolate handler SQL from authentication's earlier SQL. + // Full signed, token and browser authentication is tested separately below. + fn inner_router(&self) -> Router { + build_inner_router(self.api.clone()).layer(Extension(Self::actor())) + } + fn answer(&self, key: &str) -> serde_json::Value { + let current = self + .store + .get_ticket_assignment_tracking(WORKSPACE, &self.ticket.id, "fixture-assignment") + .unwrap() + .unwrap(); + serde_json::json!({ + "assignment_id": current.assignment_id, + "expected_last_event_id": current.last_event_id.unwrap(), + "operation_key": key, "body": "Explicit user answer: preserve this exact body" + }) + } + async fn start_orchestrator(&self) -> RuntimeWorkerRef { + // Live Runtime binding is required by the real delivery acknowledger. + let identity = (*EMBEDDED_RUNTIME_REQUEST_IDENTITY).clone(); + self.store + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), + display_name: "Embedded fixture Runtime".into(), + base_url: "in-process://embedded".into(), + public_key: identity.public_key, + public_key_fingerprint: String::new(), + binding_id: "fixture-binding".into(), + authentication_mode: StoredRuntimeAuthenticationMode::LegacyServerIssuer, + created_at: AT.into(), + updated_at: AT.into(), + revoked_at: None, + }, + false, + ) + .unwrap(); + let started = scoped_start_workspace_orchestrator( + State(self.api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: WORKSPACE.into(), + }), + ) + .await + .unwrap() + .0; + let worker = started.worker.unwrap(); + observed(self.api.host_executors.host().unwrap().drain()) + .await + .unwrap(); + observed(self.executor.drain()).await.unwrap(); + self.execution.take_inputs(); + self.execution.take_input_request_ids(); + RuntimeWorkerRef::new(worker.runtime_id, worker.worker_id) + } + async fn drain(&self) { + observed(self.api.host_executors.host().unwrap().drain()) + .await + .unwrap(); + observed(self.executor.drain()).await.unwrap(); + } + async fn finish(&self) { + // Keep the original Tokio driver alive through actual Host/cleanup work, + // then stop projection producers before the final DB barrier. + observed(self.api.host_executors.host().unwrap().drain()) + .await + .unwrap(); + observed(self.api.host_executors.cleanup().unwrap().drain()) + .await + .unwrap(); + observed(self.api.worker_projection.shutdown_async()).await; + observed(self.executor.drain()).await.unwrap(); + } + fn answer_count(&self) -> i64 { + Connection::open(&self.api.config.database_path).unwrap().query_row( + "SELECT count(*) FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2 AND kind='assignment_answer'", + rusqlite::params![WORKSPACE, self.ticket.id], |r| r.get(0)).unwrap() + } + fn assert_saved_answer(&self, key: &str, answer: &serde_json::Value) -> serde_json::Value { + let conn = Connection::open(&self.api.config.database_path).unwrap(); + let (input, source, result): (String, String, String) = conn.query_row( + "SELECT input_json,source_context_json,result_json FROM typed_ticket_assignment_tracking_operations WHERE workspace_id=?1 AND operation_key=?2", + rusqlite::params![WORKSPACE,key], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); + assert_eq!( + serde_json::from_str::(&input).unwrap()["answer"], + *answer + ); + let source: serde_json::Value = serde_json::from_str(&source).unwrap(); + assert_eq!(source["source"]["kind"], "account", "{source}"); + assert_eq!(source["source"]["user_id"], "fixture-user", "{source}"); + let receipt: serde_json::Value = serde_json::from_str(&result).unwrap(); + let (body, author, audit): (String,String,String) = conn.query_row( + "SELECT e.body,e.author,a.value FROM typed_ticket_events e JOIN typed_ticket_event_attributes a ON a.workspace_id=e.workspace_id AND a.ticket_id=e.ticket_id AND a.event_index=e.event_index AND a.key='authenticated_source' WHERE e.workspace_id=?1 AND e.ticket_id=?2 AND e.event_index=?3", + rusqlite::params![WORKSPACE,self.ticket.id,receipt["event_index"].as_u64().unwrap()], + |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); + assert_eq!(body, answer["body"].as_str().unwrap()); + assert_eq!(author, source["actor"].as_str().unwrap()); + assert_eq!( + serde_json::from_str::(&audit).unwrap(), + source + ); + receipt + } +} + +// SQLite owns the actual contention. Watchdog is a panic/hang escape, not test timing. +struct WriteLock { + release: Option>, + thread: Option>, + released: Arc, +} +impl WriteLock { + fn acquire(path: PathBuf) -> Self { + let (ready, held) = mpsc::sync_channel(1); + let (release, permit) = mpsc::channel(); + let released = Arc::new(AtomicBool::new(false)); + let flag = released.clone(); + let thread = std::thread::spawn(move || { + let conn = Connection::open(path).unwrap(); + conn.execute_batch("BEGIN IMMEDIATE").unwrap(); + ready.send(()).unwrap(); + let _ = permit.recv_timeout(WATCHDOG); + conn.execute_batch("ROLLBACK").unwrap(); + flag.store(true, Ordering::SeqCst); + }); + held.recv_timeout(WATCHDOG) + .expect("SQLite blocker did not acquire ownership"); + Self { + release: Some(release), + thread: Some(thread), + released, + } + } + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} +impl Drop for WriteLock { + fn drop(&mut self) { + self.release(); + } +} + +struct BusyGate { + entered: Option>, + resume: mpsc::Receiver<()>, +} +struct BusyObservation { + store: Arc, + gate: Box, + resume: Option>, +} +impl BusyObservation { + fn install(store: Arc) -> (Self, oneshot::Receiver<()>) { + unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 { + // SAFETY: boxed fixture userdata stays alive until mutex-serialized unregister. + let gate = unsafe { &mut *context.cast::() }; + if let Some(entered) = gate.entered.take() { + let _ = entered.send(()); + return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok()); + } + 0 + } + let (entered, waiting) = oneshot::channel(); + let (resume, permit) = mpsc::channel(); + let mut gate = Box::new(BusyGate { + entered: Some(entered), + resume: permit, + }); + store + .with_conn(|conn| { + // SAFETY: connection owns handle; Box gives userdata a stable address. + assert_eq!( + unsafe { + ffi::sqlite3_busy_handler( + conn.handle(), + Some(busy), + (&mut *gate as *mut BusyGate).cast(), + ) + }, + ffi::SQLITE_OK + ); + Ok(()) + }) + .unwrap(); + ( + Self { + store, + gate, + resume: Some(resume), + }, + waiting, + ) + } + fn resume(&mut self) { + if let Some(resume) = self.resume.take() { + let _ = resume.send(()); + } + } +} +impl Drop for BusyObservation { + fn drop(&mut self) { + self.resume(); + self.store + .with_conn(|conn| { + // SAFETY: unregister before the owned Box is freed. + unsafe { + ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut()); + } + Ok(()) + }) + .unwrap(); + let _ = &self.gate; + } +} +async fn runtime_progress_while_locked(lock: &WriteLock) { + assert_eq!( + observed(tokio::spawn(async { "independent control" })) + .await + .unwrap(), + "independent control" + ); + // This timer IS the responsiveness contract; it does not guess SQL acceptance. + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !lock.released.load(Ordering::SeqCst), + "watchdog released lock before control/timer progressed" + ); +} +async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value { + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); + if bytes.is_empty() { + // Authentication middleware deliberately returns status-only denials; + // generated handler/domain successes and errors must still be JSON. + assert!( + matches!(expected, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN), + "unexpected empty HTTP body for {expected}" + ); + return serde_json::Value::Null; + } + serde_json::from_slice(&bytes).expect("typed JSON HTTP response") +} + +// Request lifecycle source and actual retained leases, not waiter liveness. +#[derive(Default)] +struct Lifetimes { + live: Arc, + admissions: AtomicUsize, +} +struct Lease(Arc); +impl Drop for Lease { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} +impl DbOperationLifetime for Lifetimes { + fn retain(&self) -> Box { + self.admissions.fetch_add(1, Ordering::SeqCst); + self.live.fetch_add(1, Ordering::SeqCst); + Box::new(Lease(self.live.clone())) + } +} + +// Finite admitted blockers release through RAII even if an HTTP assertion fails. +struct Saturation { + releases: Vec>, +} +impl Saturation { + fn fill(executor: &DbExecutor, workers: usize, queued: usize) -> Self { + let mut releases = Vec::new(); + for _ in 0..workers { + let (started, running) = mpsc::channel(); + let (release, permit) = mpsc::channel(); + executor + .dispatch(move || { + started.send(()).unwrap(); + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + running + .recv_timeout(WATCHDOG) + .expect("admitted blocker did not start"); + releases.push(release); + } + for _ in 0..queued { + let (release, permit) = mpsc::channel(); + executor + .dispatch(move || { + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + releases.push(release); + } + Self { releases } + } +} +impl Drop for Saturation { + fn drop(&mut self) { + self.releases.clear(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn assignment_answer_sqlite_writer_wait_keeps_timer_control_and_receipt_atomic() { + for browser in [false, true] { + let fixture = Fixture::new().await; + let answer = fixture.answer("busy-answer"); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let (mut busy, waiting) = BusyObservation::install(fixture.store.clone()); + let response = tokio::spawn(fixture.inner_router().oneshot(Fixture::request( + "POST", + &fixture.path(browser, "assignment-answer"), + answer.clone(), + ))); + observed(waiting).await.unwrap(); // actual SQLite busy callback, not first poll + runtime_progress_while_locked(&lock).await; + assert!( + !response.is_finished(), + "answer responded before write ownership" + ); + assert_eq!( + fixture.answer_count(), + 0, + "uncommitted answer became visible" + ); + lock.release(); + busy.resume(); + let receipt = + response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; + fixture.drain().await; + assert_eq!( + fixture.answer_count(), + 1, + "one call commits once; no mutation retries" + ); + assert_eq!(fixture.assert_saved_answer("busy-answer", &answer), receipt); + fixture.finish().await; + } +} + +#[tokio::test(flavor = "current_thread")] +async fn saturated_db_assignment_routes_return_503_without_accepting_answer() { + let fixture = Fixture::new().await; + let answer = fixture.answer("unaccepted-answer"); + let full = Saturation::fill(&fixture.executor, 1, 32); + let lifetime = Arc::new(Lifetimes::default()); + for browser in [false, true] { + for (method, suffix, input) in [ + ( + "GET", + "assignment-tracking/assignments", + serde_json::Value::Null, + ), + ( + "GET", + "assignment-diagnostics?assignment_id=fixture-assignment", + serde_json::Value::Null, + ), + ("POST", "assignment-answer", answer.clone()), + ] { + let response = observed(crate::db_executor::with_operation_lifetime( + lifetime.clone(), + fixture.inner_router().oneshot(Fixture::request( + method, + &fixture.path(browser, suffix), + input, + )), + )) + .await + .unwrap(); + let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; + assert_eq!( + body["diagnostics"][0]["code"], "db_capacity_exceeded", + "{body}" + ); + } + } + let response = observed(fixture.inner_router().oneshot(Fixture::request( + "POST", + &format!("/api/w/{WORKSPACE}/tickets/query"), + serde_json::json!({"handling_status":"waiting"}), + ))) + .await + .unwrap(); + let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; + assert_eq!( + body["diagnostics"][0]["code"], "db_capacity_exceeded", + "{body}" + ); + assert_eq!(lifetime.live.load(Ordering::SeqCst), 0); + drop(full); + fixture.drain().await; + assert_eq!(fixture.answer_count(), 0); + assert_eq!(Connection::open(&fixture.api.config.database_path).unwrap().query_row( + "SELECT count(*) FROM typed_ticket_assignment_tracking_operations WHERE operation_key='unaccepted-answer'", [], |r| r.get::<_,i64>(0)).unwrap(), 0); + fixture.finish().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_answer_sql_job_keeps_audit_receipt_publication_and_inherited_lifecycle() { + let fixture = Fixture::new().await; + let orchestrator = fixture.start_orchestrator().await; + let answer = fixture.answer("cancelled-answer"); + let (published, publication) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + fixture.execution.before_input_returns(move |_| { + let _ = published.send(()); + permit + .recv_timeout(WATCHDOG) + .expect("publication was not released"); + }); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let (mut busy, waiting) = BusyObservation::install(fixture.store.clone()); + let lifetime = Arc::new(Lifetimes::default()); + let response = tokio::spawn(crate::db_executor::with_operation_lifetime( + lifetime.clone(), + fixture.inner_router().oneshot(Fixture::request( + "POST", + &fixture.path(true, "assignment-answer"), + answer.clone(), + )), + )); + observed(waiting).await.unwrap(); + assert!(lifetime.live.load(Ordering::SeqCst) > 0); + response.abort(); + assert!(observed(response).await.unwrap_err().is_cancelled()); + assert!( + lifetime.live.load(Ordering::SeqCst) > 0, + "waiter cancellation released accepted SQL lifecycle" + ); + // Queue capacity remains bounded after waiter loss. + let mut queued_releases: Vec> = Vec::new(); + for _ in 0..32 { + let (queued_release, queued_permit) = mpsc::channel(); + fixture + .executor + .dispatch(move || { + let _ = queued_permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + queued_releases.push(queued_release); + } + assert!(matches!( + fixture.executor.dispatch(|| ()), + Err(Error::DbCapacityExceeded) + )); + lock.release(); + busy.resume(); + drop(queued_releases); + observed(publication).await.unwrap(); // real Runtime dispatch hook reached after SQL commit + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(fixture.answer_count(), 1); + fixture.assert_saved_answer("cancelled-answer", &answer); + // Runtime wait belongs to Host: the sole DB lane remains available for callbacks. + assert_eq!( + observed( + fixture + .executor + .run(|| Ok::<_, Error>("DB callback progressed")) + ) + .await + .unwrap(), + "DB callback progressed" + ); + assert!( + lifetime.live.load(Ordering::SeqCst) > 0, + "Host publication did not inherit admitted lifecycle" + ); + assert!( + lifetime.admissions.load(Ordering::SeqCst) >= 2, + "no inherited Host admission" + ); + release.send(()).unwrap(); + fixture.drain().await; + assert_eq!( + lifetime.live.load(Ordering::SeqCst), + 0, + "actual cleanup leaked lifecycle" + ); + let inputs = fixture.execution.take_inputs(); + assert_eq!( + inputs.len(), + 1, + "cancelled HTTP answer must still publish exactly once" + ); + assert_eq!(inputs[0].0.worker_id.to_string(), orchestrator.worker_id); + assert!( + inputs[0] + .1 + .contains(fixture.ticket.resource_key.as_deref().unwrap()) + ); + assert!(inputs[0].1.contains("input:")); + assert_eq!(fixture.execution.take_input_request_ids().len(), 1); + fixture.finish().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn assignment_answer_http_success_waits_for_publication_without_holding_db_worker() { + let fixture = Fixture::new().await; + fixture.start_orchestrator().await; + let (published, publication) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + fixture.execution.before_input_returns(move |_| { + let _ = published.send(()); + permit + .recv_timeout(WATCHDOG) + .expect("publication was not released"); + }); + let answer = fixture.answer("await-publication"); + let response = tokio::spawn(build_router(fixture.api.clone()).oneshot(Fixture::request( + "POST", + &fixture.path(true, "assignment-answer"), + answer.clone(), + ))); + observed(publication).await.unwrap(); + assert!( + !response.is_finished(), + "normal HTTP waiter returned before publication completion" + ); + assert_eq!(fixture.answer_count(), 1, "publication must follow commit"); + assert_eq!( + observed(fixture.executor.run(|| Ok::<_, Error>(42))) + .await + .unwrap(), + 42, + "Runtime dispatch blocked the sole DB worker" + ); + release.send(()).unwrap(); + let receipt = response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; + fixture.drain().await; + assert_eq!( + fixture.assert_saved_answer("await-publication", &answer), + receipt + ); + assert_eq!(fixture.execution.take_inputs().len(), 1); + fixture.finish().await; +} + +// WAL readers do not wait for an IMMEDIATE writer. For read-only callers, hold +// the actual Store connection mutex instead: it is part of the same whole DB +// job as reference resolution and the page/query read (not a mock executor job). +struct ConnectionLock { + release: Option>, + thread: Option>, + released: Arc, +} +impl ConnectionLock { + fn acquire(store: Arc) -> Self { + let (ready, held) = mpsc::sync_channel(1); + let (release, permit) = mpsc::channel(); + let released = Arc::new(AtomicBool::new(false)); + let flag = released.clone(); + let thread = std::thread::spawn(move || { + store + .with_conn(|conn| { + assert_eq!(conn.query_row("SELECT 1", [], |r| r.get::<_, i64>(0))?, 1); + ready.send(()).unwrap(); + let _ = permit.recv_timeout(WATCHDOG); + Ok(()) + }) + .unwrap(); + flag.store(true, Ordering::SeqCst); + }); + held.recv_timeout(WATCHDOG) + .expect("Store connection owner did not start"); + Self { + release: Some(release), + thread: Some(thread), + released, + } + } + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} +impl Drop for ConnectionLock { + fn drop(&mut self) { + self.release(); + } +} +struct Admission(Mutex>>); +impl DbOperationLifetime for Admission { + fn retain(&self) -> Box { + if let Some(entered) = self.0.lock().unwrap().take() { + let _ = entered.send(()); + } + Box::new(()) + } +} + +#[tokio::test(flavor = "current_thread")] +async fn assignment_read_pages_and_reference_resolution_wait_off_executor_on_store_connection() { + let fixture = Fixture::new().await; + for browser in [false, true] { + for suffix in [ + "assignment-tracking/assignments?limit=1", + "assignment-diagnostics?assignment_id=fixture-assignment&limit=1", + ] { + let mut lock = ConnectionLock::acquire(fixture.store.clone()); + let (entered, admission) = oneshot::channel(); + let response = tokio::spawn(crate::db_executor::with_operation_lifetime( + Arc::new(Admission(Mutex::new(Some(entered)))), + fixture.inner_router().oneshot(Fixture::request( + "GET", + &fixture.path(browser, suffix), + serde_json::Value::Null, + )), + )); + observed(admission).await.unwrap(); + assert_eq!(observed(tokio::spawn(async { 42 })).await.unwrap(), 42); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !lock.released.load(Ordering::SeqCst), + "synchronous pre-call reference read stalled Tokio until watchdog" + ); + assert!( + !response.is_finished(), + "read bypassed held Store connection" + ); + lock.release(); + let body = + response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; + if suffix.starts_with("assignment-tracking") { + assert_eq!( + body["items"][0]["assignment_id"], "fixture-assignment", + "{body}" + ); + assert!(body["next_assignment_id"].is_null(), "{body}"); + } else { + assert_eq!(body["assignment_id"], "fixture-assignment", "{body}"); + assert_eq!(body["input_origin_known"], true, "{body}"); + assert!(body.get("payload_json").is_none(), "{body}"); + } + fixture.drain().await; + } + } + fixture.finish().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn typed_ticket_query_offloads_principal_handling_and_closure_filters_before_page_limit() { + let fixture = Fixture::new().await; + let backend = browser_ticket_backend(&fixture.api).unwrap(); + let closed = backend + .create(ticket::NewTicket::new("Closed unrelated case")) + .unwrap(); + let before = backend.show(closed.id.clone().into()).unwrap(); + backend + .update_case( + &closed.id, + ticket::TicketCaseUpdate { + expected_content_digest: ticket::ticket_content_digest(&before), + expected_state: before.meta.workflow_state, + expected_closure_operation_key: before.closure_operation_key.clone(), + operation_key: "query-close".into(), + action: ticket::TicketCaseAction::Close { + reason: ticket::TicketClosureReason::Completed, + }, + explanation: "Confirmed result".into(), + references: vec![], + author: None, + }, + ) + .unwrap(); + for n in 0..3 { + backend + .create(ticket::NewTicket::new(format!("Newer unassigned {n}"))) + .unwrap(); + } + let path = format!("/api/w/{WORKSPACE}/tickets/query"); + for (input, expected) in [ + ( + serde_json::json!({"assigned_principal":{"kind":"workspace_agent","agent_key":"workspace-orchestrator"},"handling_status":"waiting","limit":1}), + Some(fixture.ticket.id.as_str()), + ), + ( + serde_json::json!({"assigned_principal":{"kind":"workspace_agent","agent_key":"workspace-orchestrator"},"handling_status":"done","limit":1}), + None, + ), + ( + serde_json::json!({"assigned_principal":{"kind":"worker","runtime_id":"unavailable","worker_id":"unavailable"},"limit":1}), + None, + ), + ( + serde_json::json!({"closure_reason":"completed","limit":1}), + Some(closed.id.as_str()), + ), + ( + serde_json::json!({"closure_reason":"completed","handling_status":"waiting","limit":1}), + None, + ), + ] { + // Candidate SQL has its own Authority connection, including empty-result + // queries that never reach API Store enrichment. + let mut lock = ConnectionLock::acquire(Arc::new( + fixture.api.authority.candidate_query_store_for_test(), + )); + let (entered, admission) = oneshot::channel(); + let response = tokio::spawn(crate::db_executor::with_operation_lifetime( + Arc::new(Admission(Mutex::new(Some(entered)))), + fixture + .inner_router() + .oneshot(Fixture::request("POST", &path, input)), + )); + observed(admission).await.unwrap(); + assert_eq!( + observed(tokio::spawn(async { "query control" })) + .await + .unwrap(), + "query control" + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !lock.released.load(Ordering::SeqCst), + "query blocked Tokio instead of its admitted DB worker" + ); + if response.is_finished() { + lock.release(); + let response = observed(response).await.unwrap().unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + panic!( + "query returned before Store connection release: status={status}, body={}", + String::from_utf8_lossy(&body) + ); + } + lock.release(); + let body = response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; + let items = body["items"].as_array().unwrap(); + if let Some(id) = expected { + assert_eq!(items.len(), 1, "{body}"); + assert_eq!(items[0]["id"], id, "{body}"); + } else { + assert!(items.is_empty(), "{body}"); + } + fixture.drain().await; + } + // Also cross the complete browser and Account authentication boundaries. + for request in [ + Fixture::request( + "POST", + &path, + serde_json::json!({"handling_status":"waiting"}), + ), + Fixture::account_request( + "POST", + &path, + serde_json::json!({"closure_reason":"completed"}), + ), + ] { + let body = response_json( + observed(build_router(fixture.api.clone()).oneshot(request)) + .await + .unwrap(), + StatusCode::OK, + ) + .await; + assert_eq!(body["items"].as_array().unwrap().len(), 1, "{body}"); + } + fixture.finish().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn browser_and_account_assignment_routes_preserve_receipt_replay_and_map_cas_cursor_errors() { + let fixture = Fixture::new().await; + let app = build_router(fixture.api.clone()); + let answer = fixture.answer("http-replay"); + let mut saved = None; + for browser in [false, true] { + for (suffix, input) in [ + ( + "assignment-tracking/assignments?limit=1", + serde_json::Value::Null, + ), + ( + "assignment-diagnostics?assignment_id=fixture-assignment&limit=1", + serde_json::Value::Null, + ), + ] { + let path = fixture.path(browser, suffix); + let body = response_json( + observed(app.clone().oneshot(Fixture::request("GET", &path, input))) + .await + .unwrap(), + StatusCode::OK, + ) + .await; + assert!(!body.is_null()); + } + let path = fixture.path(browser, "assignment-answer"); + let response = observed(app.clone().oneshot(Fixture::request( + "POST", + &path, + answer.clone(), + ))) + .await + .unwrap(); + let receipt = response_json(response, StatusCode::OK).await; + if let Some(saved) = &saved { + assert_eq!(&receipt, saved); + } else { + saved = Some(receipt); + } + let receipt = response_json( + observed( + app.clone() + .oneshot(Fixture::account_request("POST", &path, answer.clone())), + ) + .await + .unwrap(), + StatusCode::OK, + ) + .await; + assert_eq!( + Some(receipt), + saved, + "same authenticated Account/User through token and cookie must replay" + ); + let mut changed = answer.clone(); + changed["body"] = serde_json::json!("different replay body"); + let error = response_json( + observed( + app.clone() + .oneshot(Fixture::request("POST", &path, changed)), + ) + .await + .unwrap(), + StatusCode::CONFLICT, + ) + .await; + assert!(error.to_string().contains("fingerprint"), "{error}"); + let mut stale = answer.clone(); + stale["operation_key"] = serde_json::json!(format!("stale-{browser}")); + stale["expected_last_event_id"] = serde_json::json!("unobserved-event"); + response_json( + observed(app.clone().oneshot(Fixture::request("POST", &path, stale))) + .await + .unwrap(), + StatusCode::CONFLICT, + ) + .await; + for suffix in [ + "assignment-tracking/assignments?after_assignment_id=foreign", + "assignment-diagnostics?assignment_id=fixture-assignment&after_input_event_id=ticket-event:999", + "assignment-diagnostics?assignment_id=fixture-assignment&limit=101", + ] { + response_json( + observed(app.clone().oneshot(Fixture::request( + "GET", + &fixture.path(browser, suffix), + serde_json::Value::Null, + ))) + .await + .unwrap(), + StatusCode::BAD_REQUEST, + ) + .await; + } + let mut without_origin = Fixture::request("POST", &path, answer.clone()); + without_origin.headers_mut().remove(ORIGIN); + response_json( + observed(app.clone().oneshot(without_origin)).await.unwrap(), + StatusCode::FORBIDDEN, + ) + .await; + let missing = path.replace( + fixture.ticket.resource_key.as_deref().unwrap(), + "T-999999999", + ); + response_json( + observed( + app.clone() + .oneshot(Fixture::request("POST", &missing, answer.clone())), + ) + .await + .unwrap(), + StatusCode::NOT_FOUND, + ) + .await; + let mut unauthenticated = Fixture::request("POST", &path, answer.clone()); + unauthenticated.headers_mut().remove("cookie"); + response_json( + observed(app.clone().oneshot(unauthenticated)) + .await + .unwrap(), + StatusCode::UNAUTHORIZED, + ) + .await; + } + fixture.drain().await; + assert_eq!(fixture.answer_count(), 1); + assert_eq!( + Some(fixture.assert_saved_answer("http-replay", &answer)), + saved + ); + fixture.finish().await; +} + +// Uses the same request-proof signer and registered source records as existing +// HTTP auth fixtures. Runtime identity comes only from the verified proof. +fn signed_request( + identity: &RuntimeIdentityMaterial, + worker: &str, + method: &str, + path: &str, + body: serde_json::Value, +) -> Request { + let body = if method == "GET" { + Vec::new() + } else { + serde_json::to_vec(&body).unwrap() + }; + let proof = worker_runtime::auth::RuntimeRequestSourceSigner::from_identity(identity) + .issue( + "assignment-db-server", + WORKSPACE, + Some(worker), + worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION, + method, + path, + &body, + i64::try_from(worker_runtime::auth::unix_now_seconds()).unwrap(), + 30, + ) + .unwrap(); + Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header( + worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER, + proof, + ) + .body(axum::body::Body::from(body)) + .unwrap() +} + +#[tokio::test(flavor = "current_thread")] +async fn signed_assignment_answer_rejects_non_orchestrator_and_replayed_proof_without_a_receipt() { + let fixture = Fixture::new().await; + let identity = RuntimeIdentityMaterial::generate("signed-source").unwrap(); + fixture + .store + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: "signed-source".into(), + display_name: "Signed source".into(), + base_url: "https://runtime.example.invalid".into(), + public_key: identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_id: "signed-binding".into(), + authentication_mode: StoredRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: AT.into(), + updated_at: AT.into(), + revoked_at: None, + }, + true, + ) + .unwrap(); + fixture + .store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: WORKSPACE.into(), + worker: RuntimeWorkerRef::new("signed-source", "source-worker"), + display_name: "Source worker".into(), + profile: Some("builtin:coder".into()), + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: AT.into(), + updated_at: AT.into(), + }) + .unwrap(); + let app = build_router(fixture.api.clone()); + for browser in [false, true] { + let path = fixture.path(browser, "assignment-answer"); + let answer = fixture.answer("denied-source"); + let request = signed_request(&identity, "source-worker", "POST", &path, answer.clone()); + let proof = + request.headers()[worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER].clone(); + response_json( + observed(app.clone().oneshot(request)).await.unwrap(), + StatusCode::FORBIDDEN, + ) + .await; + let mut replay = signed_request(&identity, "source-worker", "POST", &path, answer.clone()); + replay.headers_mut().insert( + worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER, + proof, + ); + response_json( + observed(app.clone().oneshot(replay)).await.unwrap(), + StatusCode::UNAUTHORIZED, + ) + .await; + // The identical registered signed Worker can read; 403 above is domain + // authorization, not accidentally failed proof authentication. + for suffix in [ + "assignment-tracking/assignments?limit=1", + "assignment-diagnostics?assignment_id=fixture-assignment", + ] { + response_json( + observed(app.clone().oneshot(signed_request( + &identity, + "source-worker", + "GET", + &fixture.path(browser, suffix), + serde_json::Value::Null, + ))) + .await + .unwrap(), + StatusCode::OK, + ) + .await; + } + } + fixture.drain().await; + assert_eq!(fixture.answer_count(), 0); + assert_eq!(Connection::open(&fixture.api.config.database_path).unwrap().query_row( + "SELECT count(*) FROM typed_ticket_assignment_tracking_operations WHERE operation_key='denied-source'",[],|r|r.get::<_,i64>(0)).unwrap(),0); + fixture.finish().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn registered_current_signed_orchestrator_answers_both_routes_with_source_bound_receipt() { + let fixture = Fixture::new().await; + let orchestrator = fixture.start_orchestrator().await; + let identity = (*EMBEDDED_RUNTIME_REQUEST_IDENTITY).clone(); + let binding_id = fixture + .api + .config_store + .get_workspace_runtime_binding(WORKSPACE, EMBEDDED_WORKER_RUNTIME_ID) + .unwrap() + .unwrap() + .binding_id; + let answer = fixture.answer("signed-current-answer"); + let mut saved = None; + let app = build_router(fixture.api.clone()); + for browser in [false, true] { + let request = signed_request( + &identity, + &orchestrator.worker_id, + "POST", + &fixture.path(browser, "assignment-answer"), + answer.clone(), + ); + let receipt = response_json( + observed(app.clone().oneshot(request)).await.unwrap(), + StatusCode::OK, + ) + .await; + if let Some(saved) = &saved { + assert_eq!(&receipt, saved); + } else { + saved = Some(receipt); + } + } + fixture.drain().await; + assert_eq!(fixture.answer_count(), 1); + assert_eq!( + fixture.execution.take_inputs().len(), + 1, + "receipt replay must not republish input" + ); + let conn = Connection::open(&fixture.api.config.database_path).unwrap(); + let source: String = conn.query_row( + "SELECT source_context_json FROM typed_ticket_assignment_tracking_operations WHERE workspace_id=?1 AND operation_key='signed-current-answer'", + [WORKSPACE],|r|r.get(0)).unwrap(); + let source: serde_json::Value = serde_json::from_str(&source).unwrap(); + assert_eq!(source["source"]["kind"], "runtime", "{source}"); + assert_eq!( + source["source"]["worker_id"], orchestrator.worker_id, + "{source}" + ); + assert_eq!( + source["source"]["authentication_binding_id"], binding_id, + "{source}" + ); + // Same operation key/body from an Account is not the signed Worker's receipt. + response_json( + observed(app.oneshot(Fixture::account_request( + "POST", + &fixture.path(true, "assignment-answer"), + answer, + ))) + .await + .unwrap(), + StatusCode::CONFLICT, + ) + .await; + assert_eq!(fixture.answer_count(), 1); + fixture.finish().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn saturated_signed_assignment_live_discovery_returns_503_before_sql_while_db_callbacks_progress() + { + let fixture = Fixture::new().await; + let orchestrator = fixture.start_orchestrator().await; + let binding_id = fixture + .api + .config_store + .get_workspace_runtime_binding(WORKSPACE, &orchestrator.runtime_id) + .unwrap() + .unwrap() + .binding_id; + let context = server_api::ServerRequestContext { + actor: None, + worker_source: Some(server_api::ServerWorkerSource { + runtime_id: orchestrator.runtime_id.clone(), + worker_id: orchestrator.worker_id.clone(), + }), + runtime_source: Some(server_api::ServerRuntimeSource { + authentication_binding_id: binding_id, + runtime_id: orchestrator.runtime_id.clone(), + worker_id: Some(orchestrator.worker_id.clone()), + }), + origin: None, + transport_headers: Vec::new(), + }; + // Isolate the generated handler's live-discovery admission from earlier + // authentication Host permits. Actual signed proof authentication is covered + // by the current-source/denial tests above; this context is the trusted input + // to the same canonical/Browser generated HTTP handlers. + let service = Arc::new(ServerApiContractService::Workspace(fixture.api.clone())); + let app = server_api::server_api_axum::ticket_assignment_answer(service.clone()) + .merge(server_api::server_api_axum::ticket_assignment_answer_browser(service)) + .layer(Extension(context)); + let full = Saturation::fill(&fixture.api.host_executors.host().unwrap(), 2, 16); + for browser in [false, true] { + let key = format!("host-denied-answer-{browser}"); + let lifetime = Arc::new(Lifetimes::default()); + let response = observed(crate::db_executor::with_operation_lifetime( + lifetime.clone(), + app.clone().oneshot(Fixture::request( + "POST", + &fixture.path(browser, "assignment-answer"), + fixture.answer(&key), + )), + )) + .await + .unwrap(); + let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; + assert_eq!( + body["diagnostics"][0]["code"], "db_capacity_exceeded", + "{body}" + ); + assert_eq!( + lifetime.admissions.load(Ordering::SeqCst), + 1, + "rejected live discovery must not proceed to SQL admission" + ); + assert_eq!( + lifetime.live.load(Ordering::SeqCst), + 0, + "rejected discovery retained a request lifecycle lease" + ); + assert_eq!( + observed( + fixture + .executor + .run(|| Ok::<_, Error>("DB callback progressed")) + ) + .await + .unwrap(), + "DB callback progressed", + "Host saturation consumed the sole DB worker" + ); + assert_eq!( + fixture.answer_count(), + 0, + "discovery rejection committed an answer/audit" + ); + assert_eq!(Connection::open(&fixture.api.config.database_path).unwrap().query_row( + "SELECT count(*) FROM typed_ticket_assignment_tracking_operations WHERE workspace_id=?1 AND operation_key=?2", + rusqlite::params![WORKSPACE, key], |r| r.get::<_, i64>(0)).unwrap(), 0, + "discovery rejection committed a receipt"); + assert!( + fixture.execution.take_inputs().is_empty(), + "unaccepted answer dispatched Runtime input" + ); + } + drop(full); + fixture.finish().await; +} diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 09a50e532..e43bac8fe 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -769,7 +769,7 @@ async fn delivery_real_fixture( DeliveryScriptedClient, ) { let config = test_server_config(root); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); @@ -1007,7 +1007,7 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi tokio::task::yield_now().await; let reopened = WorkspaceApi::new_with_execution_backend( config.clone(), - Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()), + Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()), execution.clone(), ) .await diff --git a/crates/workspace-server/src/server_db_execution_tests.rs b/crates/workspace-server/src/server_db_execution_tests.rs new file mode 100644 index 000000000..e9a6538f2 --- /dev/null +++ b/crates/workspace-server/src/server_db_execution_tests.rs @@ -0,0 +1,769 @@ +//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs). +use super::*; +use crate::SqliteWorkspaceStore; +use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime}; +use rusqlite::{Connection, ffi}; +use std::{ + ffi::c_void, + future::Future, + num::NonZeroUsize, + sync::{atomic::AtomicBool, mpsc}, + thread::JoinHandle, + time::Duration as StdDuration, +}; +use tokio::sync::oneshot; + +const WORKSPACE: &str = "db-execution"; +const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects"; +const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2); +const WATCHDOG: StdDuration = StdDuration::from_secs(5); + +async fn observed(future: impl Future) -> T { + tokio::time::timeout(OBSERVE_LIMIT, future) + .await + .expect("request did not reach the expected DB/lifecycle state") +} + +struct Fixture { + // Keep temporary authority files alive until all other fixture owners drop. + server: WorkspaceServerApi, + api: WorkspaceApi, + executor: DbExecutor, + _dir: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + // Explicit startup config: never resolve the developer's live data dir. + let config = ServerConfig { + workspace_id: WORKSPACE.into(), + workspace_display_name: "DB execution fixture".into(), + workspace_created_at: "2026-01-01T00:00:00Z".into(), + workspace_execution_root: dir.path().into(), + database_path: dir.path().join("server.db"), + data_root: dir.path().join("data"), + embedded_runtime_store_root: dir.path().join("runtime"), + static_assets_dir: None, + auth: AuthConfig::Passkey { + rp_id: "localhost".into(), + origin: "http://localhost:5173".into(), + public_base_url: "http://localhost:5173".into(), + cookie_name: "db_execution_session".into(), + }, + max_records: 100, + repositories: Vec::new(), + runtime_event_sources: Vec::new(), + remote_runtime_sources: Vec::new(), + backend_base_url: Some("http://localhost:8787".into()), + }; + let store = Arc::new( + SqliteWorkspaceStore::open_for_test(&config.database_path) + .unwrap() + .with_db_executor(executor.clone()), + ); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + store + .upsert_user(&UserRecord { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + created_at: config.workspace_created_at.clone(), + updated_at: config.workspace_created_at.clone(), + }) + .unwrap(); + store + .create_browser_session(&crate::store::BrowserSessionRecord { + session_id: "fixture-session".into(), + token_hash: crate::auth::token_hash("fixture-browser-token"), + user_id: "fixture-user".into(), + created_at: config.workspace_created_at.clone(), + expires_at: "2999-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + let api = WorkspaceApi::new_for_test(config.clone(), store.clone()) + .await + .unwrap(); + let server = WorkspaceServerApi::new_for_test(config, store); + // Cache only the real inner router; avoid starting the unrelated hook. + server + .apis + .lock() + .await + .insert(WORKSPACE.into(), api.clone()); + server + .routers + .lock() + .await + .insert(WORKSPACE.into(), build_inner_router(api.clone())); + executor.drain().await.unwrap(); + Self { + server, + api, + executor, + _dir: dir, + } + } + + fn request(method: &str, path: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header("cookie", "db_execution_session=fixture-browser-token") + .header(ORIGIN, "http://localhost:5173") + .body(axum::body::Body::from(body.to_string())) + .unwrap() + } + + fn subject_request(role: &str) -> Request { + Self::request( + "POST", + SUBJECTS, + serde_json::json!({ "role": role, "behavior_md": "" }), + ) + } + + fn dispatch(&self, request: Request) -> tokio::task::JoinHandle { + let server = self.server.clone(); + tokio::spawn(async move { dispatch_workspace_request(State(server), request).await }) + } + + // A pre-authenticated generated router isolates handler admission from the + // earlier browser-session DB read. Full authentication/dispatch is exercised + // separately by the cancellation and warm-Feature tests below. + fn inner_router(&self) -> Router { + build_inner_router(self.api.clone()).layer(Extension(RequestActor { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + auth_method: ActorAuthMethod::BrowserSession, + })) + } + + fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) { + let database = self + .api + .feature_storage + .open(&self.api.subjektiv_registration) + .unwrap(); + // Initialize the trusted scope, then obtain SQLite's actual file path + // from the managed handle rather than guessing the Feature directory. + crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap(); + let path = database + .with_connection(|conn| { + Ok(PathBuf::from(conn.query_row( + "SELECT file FROM pragma_database_list WHERE name = 'main'", + [], + |row| row.get::<_, String>(0), + )?)) + }) + .unwrap(); + (database, path) + } +} + +/// SQLite write ownership lives on an OS thread, including a bounded watchdog. +/// Drop releases and joins it even when an assertion panics on the async side. +struct WriteLock { + release: Option>, + thread: Option>, + released: Arc, +} + +impl WriteLock { + fn acquire(path: PathBuf) -> Self { + let (ready, held) = mpsc::sync_channel(1); + let (release, permit) = mpsc::channel(); + let released = Arc::new(AtomicBool::new(false)); + let flag = released.clone(); + let thread = std::thread::spawn(move || { + let conn = Connection::open(path).unwrap(); + conn.execute_batch("BEGIN IMMEDIATE").unwrap(); + ready.send(()).unwrap(); + let _ = permit.recv_timeout(WATCHDOG); + conn.execute_batch("ROLLBACK").unwrap(); + flag.store(true, Ordering::SeqCst); + }); + held.recv_timeout(WATCHDOG) + .expect("SQLite lock holder did not acquire its transaction"); + Self { + release: Some(release), + thread: Some(thread), + released, + } + } + + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} +impl Drop for WriteLock { + fn drop(&mut self) { + self.release(); + } +} + +struct BusyGate { + entered: Option>, + resume: mpsc::Receiver<()>, +} + +/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's +/// userdata API can, without process globals or changing production hooks. +enum BusyDatabase { + Feature(crate::FeatureDatabase), + Store(Arc), +} +impl BusyDatabase { + fn access(&self, access: impl FnOnce(&Connection)) { + match self { + Self::Feature(database) => database + .with_connection(|conn| { + access(conn); + Ok(()) + }) + .unwrap(), + Self::Store(store) => store + .with_conn(|conn| { + access(conn); + Ok(()) + }) + .unwrap(), + } + } +} +struct BusyObservation { + database: BusyDatabase, + gate: Box, + resume: Option>, +} +impl BusyObservation { + fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) { + Self::install_on(BusyDatabase::Feature(database)) + } + fn install_store(store: Arc) -> (Self, oneshot::Receiver<()>) { + Self::install_on(BusyDatabase::Store(store)) + } + fn install_on(database: BusyDatabase) -> (Self, oneshot::Receiver<()>) { + unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 { + // SAFETY: owned Box remains registered until Drop unregisters it, + // serialized by the managed connection mutex. + let gate = unsafe { &mut *context.cast::() }; + if let Some(entered) = gate.entered.take() { + let _ = entered.send(()); + return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok()); + } + 0 + } + let (entered, waiting) = oneshot::channel(); + let (resume, permit) = mpsc::channel(); + let mut gate = Box::new(BusyGate { + entered: Some(entered), + resume: permit, + }); + database.access(|conn| { + // SAFETY: handle belongs to this connection; the Box is stable. + let code = unsafe { + ffi::sqlite3_busy_handler( + conn.handle(), + Some(busy), + (&mut *gate as *mut BusyGate).cast(), + ) + }; + assert_eq!(code, ffi::SQLITE_OK); + }); + ( + Self { + database, + gate, + resume: Some(resume), + }, + waiting, + ) + } + fn resume(&mut self) { + if let Some(resume) = self.resume.take() { + let _ = resume.send(()); + } + } +} +impl Drop for BusyObservation { + fn drop(&mut self) { + self.resume(); + self.database.access(|conn| { + // SAFETY: unregister before the boxed userdata is freed. + unsafe { + ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut()); + } + }); + let _ = &self.gate; + } +} + +// A handler with one DB stage can expose its admission without adding a test +// hook to production. Polling its future to Pending proves run accepted it. +struct AdmissionObservation(Mutex>>); +impl DbOperationLifetime for AdmissionObservation { + fn retain(&self) -> Box { + if let Some(accepted) = self.0.lock().unwrap().take() { + let _ = accepted.send(()); + } + Box::new(()) + } +} +fn admission_observation() -> (Arc, oneshot::Receiver<()>) { + let (accepted, receiver) = oneshot::channel(); + ( + Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))), + receiver, + ) +} + +struct StateCheckSaturation { + executor: DbExecutor, + stages: std::sync::atomic::AtomicUsize, + releases: Arc>>>, +} +impl DbOperationLifetime for StateCheckSaturation { + fn retain(&self) -> Box { + if self.stages.fetch_add(1, Ordering::SeqCst) == 1 { + // Authentication has completed; saturate just before the lifecycle + // lookup is admitted. The callback's own dispatch admissions are + // distinct stages and do not recurse into this branch. + let (started, running) = mpsc::channel(); + let (release, permit) = mpsc::channel(); + self.executor + .dispatch(move || { + let _ = started.send(()); + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + running + .recv_timeout(WATCHDOG) + .expect("state-check blocker did not start"); + self.releases.lock().unwrap().push(release); + let (release, permit) = mpsc::channel(); + self.executor + .dispatch(move || { + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + self.releases.lock().unwrap().push(release); + } + Box::new(()) + } +} +struct ReleaseStateCheckBlockers(Arc>>>); +impl Drop for ReleaseStateCheckBlockers { + fn drop(&mut self) { + self.0.lock().unwrap().clear(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() { + let fixture = Fixture::new().await; + let releases = Arc::new(Mutex::new(Vec::new())); + let release = ReleaseStateCheckBlockers(releases.clone()); + let source = Arc::new(StateCheckSaturation { + executor: fixture.executor.clone(), + stages: std::sync::atomic::AtomicUsize::new(0), + releases, + }); + let router = Router::new() + .route( + SUBJECTS, + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .layer(axum::middleware::from_fn_with_state( + fixture.api.clone(), + authorize_workspace_api_request, + )); + let response = observed(crate::db_executor::with_operation_lifetime( + source, + router.oneshot(Fixture::subject_request("not-executed")), + )) + .await + .unwrap(); + drop(release); + observed(fixture.executor.drain()).await.unwrap(); + let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; + assert!( + !body.to_string().contains("Workspace is deleting"), + "{body}" + ); +} + +async fn runtime_progress_while_locked(lock: &WriteLock) { + let control = tokio::spawn(async { "control progressed" }); + assert_eq!(observed(control).await.unwrap(), "control progressed"); + // The timer is the contract, not a guessed wait for request completion. + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !lock.released.load(Ordering::SeqCst), + "SQLite watchdog released the lock before runtime progress" + ); +} +async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value { + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); + serde_json::from_slice(&bytes).expect("typed JSON response") +} +fn subject_count(path: &Path, role: &str) -> i64 { + Connection::open(path) + .unwrap() + .query_row( + "SELECT count(*) FROM subjects WHERE role = ?1", + [role], + |row| row.get(0), + ) + .unwrap() +} + +#[tokio::test(flavor = "current_thread")] +async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() { + let fixture = Fixture::new().await; + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let (lifetime, accepted) = admission_observation(); + let request = Fixture::request( + "POST", + "/api/w/db-execution/tickets", + serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(), + ); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture.inner_router().oneshot(request), + )); + assert!(futures::poll!(&mut response).is_pending()); + observed(accepted).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + futures::poll!(&mut response).is_pending(), + "mutation responded before the SQLite lock was released" + ); + let reader = Connection::open(&fixture.api.config.database_path).unwrap(); + let count: i64 = reader + .query_row( + "SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1", + [WORKSPACE], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 0, "Ticket must not be visible before commit"); + lock.release(); + let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; + assert!(body["id"].is_string(), "{body}"); + // Ticket dispatch can call Runtime callbacks, so its operation owner is + // the separate bounded host executor, not the store executor. + observed(fixture.api.host_executors.host().unwrap().drain()) + .await + .unwrap(); + fixture.executor.drain().await.unwrap(); + let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap(); + assert_eq!(count, 1, "one accepted HTTP mutation must commit once"); +} + +#[tokio::test(flavor = "current_thread")] +async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + drop(database); // FeatureStorage retains Weak handles: the route must cold-open. + let mut lock = WriteLock::acquire(path); + let (lifetime, accepted) = admission_observation(); + let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture.inner_router().oneshot(request), + )); + assert!(futures::poll!(&mut response).is_pending()); + observed(accepted).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + futures::poll!(&mut response).is_pending(), + "cold open must wait for its SQLite scope transaction" + ); + lock.release(); + let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; + assert_eq!(body["items"], serde_json::json!([]), "{body}"); + fixture.executor.drain().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let response = fixture.dispatch(Fixture::subject_request("locked-subject")); + observed(waiting).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + !response.is_finished(), + "HTTP response must not publish an uncommitted subject" + ); + assert_eq!(subject_count(&path, "locked-subject"), 0); + lock.release(); + busy.resume(); + let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await; + assert_eq!(body["role"], "locked-subject", "{body}"); + assert!( + body["memory_fingerprint"].is_string(), + "response assembly must finish in the DB operation: {body}" + ); + fixture.executor.drain().await.unwrap(); + assert_eq!(subject_count(&path, "locked-subject"), 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let response = fixture.dispatch(Fixture::subject_request("cancelled-subject")); + observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling. + response.abort(); + assert!(observed(response).await.unwrap_err().is_cancelled()); + let admission = fixture.server.admission_for_workspace(WORKSPACE).await; + let mut deletion = Box::pin(admission.lock_deletion()); + assert!( + futures::poll!(&mut deletion).is_pending(), + "aborting HTTP must not drop the accepted SQL RequestLease" + ); + assert!( + admission.admit(false).is_none(), + "deletion must fence fresh requests" + ); + assert_eq!(subject_count(&path, "cancelled-subject"), 0); + lock.release(); + busy.resume(); + fixture.executor.drain().await.unwrap(); + let deletion = observed(deletion).await; + assert_eq!( + subject_count(&path, "cancelled-subject"), + 1, + "cancelled waiter must neither cancel nor retry accepted SQL" + ); + drop(deletion); + assert!( + admission.admit(false).is_some(), + "deletion lease cleanup must reopen admission" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let running = fixture.dispatch(Fixture::subject_request("running-subject")); + observed(waiting).await.unwrap(); + let (lifetime, accepted) = admission_observation(); + let mut queued = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture + .inner_router() + .oneshot(Fixture::subject_request("queued-subject")), + )); + assert!(futures::poll!(&mut queued).is_pending()); + observed(accepted).await.unwrap(); + drop(queued); // Accepted job still owns the only waiting slot. + let rejected = observed( + fixture + .inner_router() + .oneshot(Fixture::subject_request("rejected-subject")), + ) + .await + .unwrap(); + let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await; + let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap(); + assert!( + error.message.contains("capacity"), + "typed overload error: {body}" + ); + assert_eq!(subject_count(&path, "queued-subject"), 0); + assert_eq!(subject_count(&path, "rejected-subject"), 0); + let mut drained = Box::pin(fixture.executor.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + lock.release(); + busy.resume(); + response_json(observed(running).await.unwrap(), StatusCode::CREATED).await; + observed(drained).await.unwrap(); + assert_eq!( + subject_count(&path, "queued-subject"), + 1, + "cancelled accepted mutation persists exactly once" + ); + assert_eq!( + subject_count(&path, "rejected-subject"), + 0, + "overload must not execute rejected SQL" + ); + let retry = fixture.dispatch(Fixture::subject_request("after-drain")); + response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await; + fixture.executor.drain().await.unwrap(); + assert_eq!( + subject_count(&path, "after-drain"), + 1, + "drain must release actual-job capacity" + ); +} + +#[tokio::test(start_paused = true)] +async fn saturated_cancelled_host_jobs_keep_their_budget_but_not_other_fixture_or_cleanup_admission() + { + use std::sync::{ + Condvar, + atomic::{AtomicUsize, Ordering}, + }; + // A drop guard releases workers even when an assertion panics. + struct Release(Arc<(Mutex, Condvar)>); + impl Drop for Release { + fn drop(&mut self) { + *self.0.0.lock().unwrap() = true; + self.0.1.notify_all(); + } + } + let owner = HostExecutors::default(); + let other = HostExecutors::default(); + let host = owner.host().unwrap(); + let gate = Arc::new((Mutex::new(false), Condvar::new())); + let release = Release(gate.clone()); + let completed = Arc::new(AtomicUsize::new(0)); + let mut waiters = Vec::new(); + for _ in 0..2 { + let host = host.clone(); + let gate = gate.clone(); + let completed = completed.clone(); + let (started, accepted) = tokio::sync::oneshot::channel(); + waiters.push(tokio::spawn(async move { + host.run(move || { + started.send(()).unwrap(); + let mut released = gate.0.lock().unwrap(); + while !*released { + released = gate.1.wait(released).unwrap(); + } + completed.fetch_add(1, Ordering::SeqCst); + Ok::<_, Error>(()) + }) + .await + })); + accepted.await.unwrap(); + } + for _ in 0..16 { + let completed = completed.clone(); + host.dispatch(move || { + completed.fetch_add(1, Ordering::SeqCst); + }) + .unwrap(); + } + for waiter in waiters { + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + } + assert!(matches!( + host.run(|| Ok::<_, Error>(())).await, + Err(Error::DbCapacityExceeded) + )); + // Paused Tokio time can advance without completing or releasing actual Host + // work. Neither another fixture nor cleanup needs this saturated Host lane. + tokio::time::advance(std::time::Duration::from_secs(31)).await; + assert_eq!(completed.load(Ordering::SeqCst), 0); + other + .host() + .unwrap() + .run(|| Ok::<_, Error>(())) + .await + .unwrap(); + owner + .cleanup() + .unwrap() + .run(|| Ok::<_, Error>(())) + .await + .unwrap(); + assert!(matches!( + host.run(|| Ok::<_, Error>(())).await, + Err(Error::DbCapacityExceeded) + )); + drop(release); + host.drain().await.unwrap(); + assert_eq!(completed.load(Ordering::SeqCst), 18); + host.run(|| Ok::<_, Error>(())).await.unwrap(); + host.drain().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn restore_intent_pins_once_after_write_ownership_without_a_deferred_read_upgrade() { + let fixture = Fixture::new().await; + let store = fixture.api.config_store.clone(); + let (mut busy, waiting) = BusyObservation::install_store(store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let worker = RuntimeWorkerRef::new("restore-runtime", "restore-worker"); + let pin_store = store.clone(); + let pin_worker = worker.clone(); + let mut pinned = Box::pin(fixture.executor.run(move || { + pin_store.pin_internal_worker_restore_intent( + WORKSPACE, + &pin_worker, + "restore-domain", + Some("observed-before-dispatch"), + Some("restore-request"), + ) + })); + tokio::select! { + result = &mut pinned => panic!("restore intent must wait for write ownership before reading, not fail a deferred read upgrade: {result:?}"), + arrived = observed(waiting) => arrived.unwrap(), + } + runtime_progress_while_locked(&lock).await; + assert!(futures::poll!(&mut pinned).is_pending()); + lock.release(); + busy.resume(); + let request = observed(pinned).await.unwrap(); + assert_eq!(request.request_id, "restore-request"); + assert_eq!( + request.expected_observation_token, + "observed-before-dispatch" + ); + fixture.executor.drain().await.unwrap(); + let saved = store + .get_internal_worker_restore_intent(WORKSPACE, &worker, &request.request_id) + .unwrap() + .unwrap(); + assert_eq!(saved.request_id, request.request_id); + assert_eq!( + saved.expected_observation_token, + request.expected_observation_token + ); + let count: i64 = store + .with_conn(|conn| { + Ok(conn.query_row( + "SELECT count(*) FROM worker_restore_intents WHERE workspace_id=?1 AND request_id=?2", + rusqlite::params![WORKSPACE, request.request_id], |row| row.get(0) + )?) + }) + .unwrap(); + assert_eq!( + count, 1, + "one admitted intent must persist exactly once, with no operation retry" + ); +} diff --git a/crates/workspace-server/src/server_drive.rs b/crates/workspace-server/src/server_drive.rs index 57c79ef7f..33bb2e908 100644 --- a/crates/workspace-server/src/server_drive.rs +++ b/crates/workspace-server/src/server_drive.rs @@ -51,14 +51,13 @@ async fn blocking( mutation: bool, action: impl FnOnce() -> workspace_drive::Result + Send + 'static, ) -> DriveResult { - tokio::task::spawn_blocking(action) + crate::db_executor::DbExecutor::shared() + .run(move || Ok::<_, Error>(action())) .await - .map_err(|_| { - if mutation { - DriveApiError::new(DriveApiErrorCode::OutcomeUnknown) - } else { - failure(503) - } + .map_err(|error| match error { + Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503), + _ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown), + _ => failure(503), })? .map_err(|error| domain_error(error, mutation)) } diff --git a/crates/workspace-server/src/server_drive_grants.rs b/crates/workspace-server/src/server_drive_grants.rs index 9d85eaf67..4d1f82fe5 100644 --- a/crates/workspace-server/src/server_drive_grants.rs +++ b/crates/workspace-server/src/server_drive_grants.rs @@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into) -> DriveApiError { }) } fn mutation_error(error: Error) -> DriveApiError { - if matches!(error, Error::Sqlite(_) | Error::Store(_)) { + if matches!( + error, + Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown + ) { fail(DriveApiErrorCode::OutcomeUnknown) } else { pre_error(error) @@ -97,23 +100,27 @@ pub(super) async fn authorized_drive( }; // Reject missing/read-only grants before upload body reception. This check // is admission only: every later operation rechecks under its own lock. - tokio::task::spawn_blocking(move || { - drive.check_authorized(write)?; - Ok::<_, workspace_drive::Error>(drive) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? - .map_err(|error| { - fail(match error { - workspace_drive::Error::Denied => DriveApiErrorCode::Denied, - workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { - DriveApiErrorCode::Conflict - } - workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, - workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, - _ => DriveApiErrorCode::StorageUnavailable, + api.store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + drive.check_authorized(write)?; + Ok::<_, workspace_drive::Error>(drive) + })()) + }) + .await + .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? + .map_err(|error| { + fail(match error { + workspace_drive::Error::Denied => DriveApiErrorCode::Denied, + workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { + DriveApiErrorCode::Conflict + } + workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, + workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, + _ => DriveApiErrorCode::StorageUnavailable, + }) }) - }) } pub(super) async fn create_grant( @@ -152,9 +159,10 @@ pub(super) async fn create_grant( revoked_at: None, }; let store = api.store.clone(); - tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant)) + api.store + .db_executor() + .run(move || store.create_workspace_drive_grant(&grant)) .await - .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? .map_err(mutation_error) } @@ -175,13 +183,12 @@ pub(super) async fn revoke_grant( actor.account_id.clone(), ); let store = api.store.clone(); - tokio::task::spawn_blocking(move || { - store.revoke_workspace_drive_grant(&workspace, &grant, &actor) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? - .map_err(mutation_error)? - .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) + api.store + .db_executor() + .run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor)) + .await + .map_err(mutation_error)? + .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) } fn validate_grant_id(id: &str) -> DriveResult<()> { @@ -212,12 +219,14 @@ pub(super) async fn list_grants( } let workspace = workspace_id.to_owned(); let store = api.store.clone(); - let mut grants = tokio::task::spawn_blocking(move || { - store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? - .map_err(pre_error)?; + let mut grants = api + .store + .db_executor() + .run(move || { + store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) + }) + .await + .map_err(pre_error)?; let more = grants.len() > limit; grants.truncate(limit); let next_after = if more { diff --git a/crates/workspace-server/src/server_ticket_case_signed_tests.rs b/crates/workspace-server/src/server_ticket_case_signed_tests.rs index 78ba317b2..8f7c38b52 100644 --- a/crates/workspace-server/src/server_ticket_case_signed_tests.rs +++ b/crates/workspace-server/src/server_ticket_case_signed_tests.rs @@ -724,7 +724,7 @@ async fn catalog_case_dispatch_authenticates_and_routes_the_canonical_workspace_ reason: ticket::TicketClosureReason::Completed, }, ); - let catalog = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); + let catalog = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); // Retain only this in-process fixture: do not create a real Runtime or a // background Orchestrator hook while testing catalog authentication/routing. catalog diff --git a/crates/workspace-server/src/server_ticket_tags_tests.rs b/crates/workspace-server/src/server_ticket_tags_tests.rs index b636a8356..1608982bd 100644 --- a/crates/workspace-server/src/server_ticket_tags_tests.rs +++ b/crates/workspace-server/src/server_ticket_tags_tests.rs @@ -529,7 +529,7 @@ async fn runtime_tag_mutations_reject_invalid_proofs_and_changed_binding_authori *tampered.body_mut() = Body::from(format!("{} ", input)); runtime_tag_response(app.clone(), tampered, StatusCode::UNAUTHORIZED).await; } - let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let original = api .store .get_workspace_runtime_binding(TEST_WORKSPACE_ID, "runtime-tags") diff --git a/crates/workspace-server/src/server_workspace_config.rs b/crates/workspace-server/src/server_workspace_config.rs index 0e0a54959..f1372b1e4 100644 --- a/crates/workspace-server/src/server_workspace_config.rs +++ b/crates/workspace-server/src/server_workspace_config.rs @@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into) -> Failure { fn mutation_error(error: Error) -> Failure { // Storage/service failures at a mutation boundary may include a commit // acknowledgement failure. Never describe those as a guaranteed rollback. - if matches!(&error, Error::Sqlite(_) | Error::Store(_)) { + if matches!( + &error, + Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown + ) { unknown() } else { pre_error(error) @@ -186,9 +189,23 @@ pub(super) async fn get( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> ConfigResult> { - let _guard = lock(api, worker).await?; - let grant = grant_for_worker(api, worker)?; - Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true))) + let guard = lock(api, worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let worker = worker.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + let grant = grant_for_worker(&api, &worker)?; + Ok::<_, Failure>( + active_link(&api, &worker, &grant, None)? + .map(|link| attachment(&grant, &link, true)), + ) + })()) + }) + .await + .map_err(pre_error)? } pub(super) async fn attach( api: &WorkspaceApi, @@ -198,46 +215,59 @@ pub(super) async fn attach( if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) { return Err(fail(400, "invalid_alias")); } - let _guard = lock(api, worker).await?; - let grant = grant_for_worker(api, worker)?; - let access = request.access.unwrap_or(grant.access); - if access == Access::ReadWrite && grant.access != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - if let Some(mut link) = active_link(api, worker, &grant, None)? { - // Same active alias/identity is idempotent; explicit write never silently downgrades. - if request.access == Some(Access::ReadWrite) - && effective_access(&grant, &link) != Access::ReadWrite - { - return Err(fail(403, "access_denied")); - } - if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly { - link.capabilities = access_capabilities(access); - link = api - .store - .attach_worker_workdir(&link) - .map_err(mutation_error)?; - } - return Ok(attachment(&grant, &link, true)); - } - let link = api - .store - .attach_worker_workdir(&WorkerWorkdirLinkRecord { - connection_id: String::new(), - workspace_id: grant.workspace_id.clone(), - worker: worker.clone(), - workdir_id: grant.working_directory_id.clone(), - alias: ALIAS.into(), - capabilities: access_capabilities(access), - linked_at: now_registry_timestamp(), - unlinked_at: None, + let guard = lock(api, worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let worker = worker.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + let grant = grant_for_worker(&api, &worker)?; + let access = request.access.unwrap_or(grant.access); + if access == Access::ReadWrite && grant.access != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + if let Some(mut link) = active_link(&api, &worker, &grant, None)? { + // Same active alias/identity is idempotent; explicit write never silently downgrades. + if request.access == Some(Access::ReadWrite) + && effective_access(&grant, &link) != Access::ReadWrite + { + return Err(fail(403, "access_denied")); + } + if access == Access::ReadOnly + && effective_access(&grant, &link) != Access::ReadOnly + { + link.capabilities = access_capabilities(access); + link = api + .store + .attach_worker_workdir(&link) + .map_err(mutation_error)?; + } + return Ok(attachment(&grant, &link, true)); + } + let link = api + .store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: grant.workspace_id.clone(), + worker: worker.clone(), + workdir_id: grant.working_directory_id.clone(), + alias: ALIAS.into(), + capabilities: access_capabilities(access), + linked_at: now_registry_timestamp(), + unlinked_at: None, + }) + .map_err(mutation_error)?; + // No Runtime filesystem/session synchronization for a logical config target. + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(attachment(&grant, &link, false)) + })()) }) - .map_err(mutation_error)?; - // No Runtime filesystem/session synchronization for a logical config target. - api.worker_projection - .refresh(worker) - .map_err(|_| unknown())?; - Ok(attachment(&grant, &link, false)) + .await + .map_err(mutation_error)? } fn resolve( api: &WorkspaceApi, @@ -313,9 +343,22 @@ pub(super) async fn observe( for path in &request.paths { path_valid(path, true)?; } - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, _guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -427,9 +470,22 @@ pub(super) async fn read( request: WorkspaceConfigReadRequest, ) -> ConfigResult { path_valid(&request.path, false)?; - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, _guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -482,12 +538,25 @@ pub(super) async fn commit( server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?, } } - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - if effective_access(&grant, &link) != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + if effective_access(&grant, &link) != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -501,17 +570,19 @@ pub(super) async fn commit( // Evaluation/semantic validation are pure preparation. Bound this stage // without ever dispatching persistence in a timed-out background task. let preparation_api = api.clone(); - let preparation = tokio::task::spawn_blocking(move || { - prepare_workspace_config_tree( + let executor = api.store.db_executor(); + let preparation = executor.run(move || { + // Timeout/cancellation releases exclusion only after actual preparation ends. + let candidate = prepare_workspace_config_tree( &preparation_api, &preparation_api.config.workspace_id, &canonical, - ) + )?; + Ok::<_, Error>((candidate, guard)) }); - let candidate = tokio::time::timeout(DEADLINE, preparation) + let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation) .await .map_err(|_| fail(503, "evaluation_timeout"))? - .map_err(|_| fail(500, "evaluation_unavailable"))? .map_err(pre_error)?; if candidate.base_digest != state.snapshot.digest || candidate.base_toolchain_fingerprint != state.contract.fingerprint @@ -520,15 +591,78 @@ pub(super) async fn commit( { return Err(fail(409, "stale_validator")); } - // Recheck durable authority before the first auxiliary/canonical effects. - let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?; - if current_grant != grant || current_link.capabilities != link.capabilities { - return Err(fail(409, "authority_changed")); + let persistence_api = api.clone(); + let worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let expected_grant = grant.clone(); + let expected_link = link.clone(); + let (state, publication) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (current_grant, current_link) = + resolve(&persistence_api, &worker, &connection_id)?; + if current_grant != expected_grant + || current_link.capabilities != expected_link.capabilities + { + return Err(fail(409, "authority_changed")); + } + // Preserve the shared helper's ordering and atomic CAS, but never + // run its remote Runtime publication on the sole DB worker. + project_repository_access_candidate( + &*persistence_api.store, + &persistence_api.repository_secrets, + &persistence_api.config.workspace_id, + &candidate, + ) + .map_err(|_| unknown())?; + let state = persistence_api + .config_store + .commit_evaluated_workspace_config( + &persistence_api.config.workspace_id, + &candidate, + ) + .map_err(|_| unknown())?; + let publication = persistence_api + .prompt_projection_cache + .resolve(&persistence_api.config.workspace_id, &state) + .ok() + .and_then(|projection| { + // Dispatch from the actual persistence owner even if its + // waiter was cancelled. Runtime I/O owns exclusion, not + // a DB worker or an open SQL transaction. + let (complete, publication) = tokio::sync::oneshot::channel(); + let runtime = persistence_api.runtime.clone(); + let admission = persistence_api.host_executors.host().and_then(|executor| { + executor.dispatch(move || { + let _guard = guard; + let _diagnostics = runtime + .observe_workspace_prompt_projection((*projection).clone()); + let _ = complete.send(()); + }) + }); + match admission { + Ok(()) => Some(publication), + Err(error) => { + tracing::warn!( + workspace_id = %persistence_api.config.workspace_id, + %error, + "Workspace config committed but Runtime publication was not admitted" + ); + None + } + } + }); + Ok((state, publication)) + })()) + }) + .await + .map_err(mutation_error)??; + if let Some(publication) = publication { + // Publication is best effort and cannot turn an acknowledged CAS into an error. + let _ = publication.await; } - // From this boundary repository credential projection may have side effects. - // Never falsely claim rollback or automatically replay a failed/uncertain save. - let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate) - .map_err(|_| unknown())?; Ok(WorkspaceConfigCommitResponse { validator: validator(&grant, &link, &state, &schema_fingerprint, ""), digest: state.snapshot.digest, @@ -544,37 +678,61 @@ pub(super) async fn create_grant( .await .map_err(pre_error)?; let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id); - let _guard = lock(api, &worker).await?; - let summary = api - .runtime - .worker(&worker) - .map_err(|_| fail(404, "worker_not_found"))?; + let guard = lock(api, &worker).await?; + let runtime = api.runtime.clone(); + let lookup_worker = worker.clone(); + let (summary, guard) = api + .host_executors + .host() + .map_err(pre_error)? + .run(move || { + Ok::<_, Error>((|| { + let summary = runtime + .worker(&lookup_worker) + .map_err(|_| fail(404, "worker_not_found"))?; + Ok::<_, Failure>((summary, guard)) + })()) + }) + .await + .map_err(pre_error)??; if summary.workspace.workspace_id.as_deref() != Some(workspace_id) { return Err(fail(403, "access_denied")); } - if let Some(grant) = api - .store - .current_workspace_config_grant(workspace_id, &worker) - .map_err(pre_error)? - { - if grant.access != request.access { - return Err(fail(409, "grant_exists_revoke_first")); - } - return Ok(grant); - } - let grant = WorkspaceConfigGrantResponse { - grant_id: Uuid::now_v7().to_string(), - workspace_id: workspace_id.into(), - runtime_id: worker.runtime_id, - worker_id: worker.worker_id, - working_directory_id: Uuid::now_v7().to_string(), - access: request.access, - revoked: false, - }; - api.store - .create_workspace_config_grant(&grant, &actor.account_id) - .map_err(mutation_error)?; - Ok(grant) + let executor = api.store.db_executor(); + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let account_id = actor.account_id.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + if let Some(grant) = api + .store + .current_workspace_config_grant(&workspace_id, &worker) + .map_err(pre_error)? + { + if grant.access != request.access { + return Err(fail(409, "grant_exists_revoke_first")); + } + return Ok(grant); + } + let grant = WorkspaceConfigGrantResponse { + grant_id: Uuid::now_v7().to_string(), + workspace_id, + runtime_id: worker.runtime_id, + worker_id: worker.worker_id, + working_directory_id: Uuid::now_v7().to_string(), + access: request.access, + revoked: false, + }; + api.store + .create_workspace_config_grant(&grant, &account_id) + .map_err(mutation_error)?; + Ok(grant) + })()) + }) + .await + .map_err(mutation_error)? } pub(super) async fn revoke_grant( api: &WorkspaceApi, @@ -585,39 +743,60 @@ pub(super) async fn revoke_grant( require_workspace_owner(api, workspace_id, actor, "Workspace config grants") .await .map_err(pre_error)?; + let lookup_api = api.clone(); + let lookup_workspace_id = workspace_id.to_owned(); + let lookup_grant_id = grant_id.to_owned(); let mut grant = api .store - .get_workspace_config_grant(workspace_id, grant_id) + .db_executor() + .run(move || { + lookup_api + .store + .get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id) + }) + .await .map_err(pre_error)? .ok_or_else(|| fail(404, "not_found"))?; let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id); - let _guard = lock(api, &worker).await?; - api.store - .revoke_workspace_config_grant(workspace_id, grant_id) - .map_err(mutation_error)?; - grant.revoked = true; - // Revocation is durable before unlink; a failed cleanup cannot restore access. - if let Some(link) = api - .store - .list_workdir_worker_links(workspace_id, &grant.working_directory_id) - .map_err(|_| unknown())? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.worker == worker) - { - api.store - .detach_worker_workdir_connection( - workspace_id, - &worker, - &link.alias, - &link.connection_id, - &now_registry_timestamp(), - ) - .map_err(|_| unknown())?; - } - api.worker_projection - .refresh(&worker) - .map_err(|_| unknown())?; - Ok(grant) + let guard = lock(api, &worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let grant_id = grant_id.to_owned(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + api.store + .revoke_workspace_config_grant(&workspace_id, &grant_id) + .map_err(mutation_error)?; + grant.revoked = true; + // Revocation is durable before unlink; a failed cleanup cannot restore access. + if let Some(link) = api + .store + .list_workdir_worker_links(&workspace_id, &grant.working_directory_id) + .map_err(|_| unknown())? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.worker == worker) + { + api.store + .detach_worker_workdir_connection( + &workspace_id, + &worker, + &link.alias, + &link.connection_id, + &now_registry_timestamp(), + ) + .map_err(|_| unknown())?; + } + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(grant) + })()) + }) + .await + .map_err(mutation_error)? } pub(super) fn summary( @@ -655,7 +834,11 @@ pub(super) fn summary( mod tests { use super::*; #[test] - fn workspace_config_mutation_storage_failures_are_conservatively_unknown() { + fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() { + assert_eq!( + mutation_error(Error::DbOperationOutcomeUnknown).classification, + Classification::Unknown + ); assert_eq!( mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification, Classification::Unknown diff --git a/crates/workspace-server/src/server_workspace_config_tests.rs b/crates/workspace-server/src/server_workspace_config_tests.rs index ac11e7766..1c932be1a 100644 --- a/crates/workspace-server/src/server_workspace_config_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_tests.rs @@ -318,7 +318,7 @@ mod workspace_config_integration { }; for scoped in [false, true] { let app = if scoped { - workspace_server_router(WorkspaceServerApi::new( + workspace_server_router(WorkspaceServerApi::new_for_test( api.config.clone(), api.store.clone(), )) @@ -803,9 +803,9 @@ mod workspace_config_integration { .supports(workdir::WorkdirSessionCapability::Command) ); let lock = current_worker_session_lock(api, &fixture.worker); - let _guard = lock.lock().await; + let _guard = Arc::new(lock.lock_owned().await); assert!( - open_current_worker_workdir_session_locked(api, &fixture.worker, &link) + open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard) .await .is_err() ); @@ -1385,7 +1385,7 @@ mod workspace_config_integration { let api = &fixture.api; let granted = grant(api, &fixture.worker, Access::ReadWrite).await; let attached = attach(api, &fixture.worker).await; - let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); assert_eq!( restarted .get_workspace_config_grant(TEST_WORKSPACE_ID, &granted.grant_id) diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index b5a229f55..1e989edd8 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -800,7 +800,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w let base = format!("http://{addr}"); let mut config = test_server_config(root.path()); config.backend_base_url = Some("server-test".into()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let captured = Arc::new(Mutex::new(None)); let abort = Arc::new(Mutex::new(None)); let factory = ConfigCaptureFactory { diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 3e44c9081..4585493d5 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -1131,6 +1131,10 @@ impl WorkspaceResourceKind { #[async_trait] pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { + fn db_executor(&self) -> crate::db_executor::DbExecutor { + crate::db_executor::DbExecutor::shared() + } + async fn schema_version(&self) -> Result; fn resource_key( &self, @@ -2257,6 +2261,7 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { #[derive(Clone)] pub struct SqliteWorkspaceStore { conn: Arc>, + db_executor: crate::db_executor::DbExecutor, } impl SqliteWorkspaceStore { @@ -2297,12 +2302,50 @@ impl SqliteWorkspaceStore { } pub fn from_connection(conn: Connection) -> Result { + Self::from_connection_with_executor(conn, crate::db_executor::DbExecutor::shared()) + } + + fn from_connection_with_executor( + conn: Connection, + db_executor: crate::db_executor::DbExecutor, + ) -> Result { prepare_connection(&conn)?; Ok(Self { conn: Arc::new(Mutex::new(conn)), + db_executor, }) } + // Ordinary fixtures own the same finite budget as production, not another + // fixture's admission/SQL/drain lifetime or a permanent global worker. + #[cfg(test)] + pub(crate) fn open_for_test(path: impl AsRef) -> Result { + Self::from_connection_for_test(Connection::open(path)?) + } + + #[cfg(test)] + pub(crate) fn in_memory_for_test() -> Result { + Self::from_connection_for_test(Connection::open_in_memory()?) + } + + #[cfg(test)] + pub(crate) fn from_connection_for_test(conn: Connection) -> Result { + Self::from_connection_with_executor( + conn, + crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits::default())?, + ) + } + + /// Override execution limits for a hermetic fixture; subsequent clones share it. + pub fn with_db_executor(mut self, executor: crate::db_executor::DbExecutor) -> Self { + self.db_executor = executor; + self + } + + pub fn db_executor(&self) -> crate::db_executor::DbExecutor { + self.db_executor.clone() + } + pub(crate) fn with_conn(&self, f: impl FnOnce(&Connection) -> Result) -> Result { let conn = self .conn @@ -2322,12 +2365,16 @@ impl SqliteWorkspaceStore { f(&mut conn) } - pub(crate) fn with_transaction( + pub(crate) fn with_immediate_transaction( &self, f: impl FnOnce(&rusqlite::Transaction<'_>) -> Result, ) -> Result { self.with_conn_mut(|conn| { - let tx = conn.transaction()?; + // These callers read authority/journal state before mutating it. + // Acquire write ownership before the first read: a DEFERRED read + // upgrade against another connection's writer can return BUSY + // without invoking the busy handler. Never retry the domain mutation. + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let value = f(&tx)?; tx.commit()?; Ok(value) @@ -2388,7 +2435,7 @@ impl SqliteWorkspaceStore { observed_token: Option<&str>, explicit_request_id: Option<&str>, ) -> Result { - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { let existing: Option<(String, String)> = if let Some(id) = explicit_request_id { tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4", params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()? @@ -3872,8 +3919,16 @@ fn insert_workspace_runtime_binding_audit( #[async_trait] impl ControlPlaneStore for SqliteWorkspaceStore { + fn db_executor(&self) -> crate::db_executor::DbExecutor { + self.db_executor.clone() + } + async fn schema_version(&self) -> Result { - self.with_conn(current_schema_version) + let store = self.clone(); + + self.db_executor + .run(move || store.with_conn(current_schema_version)) + .await } fn resource_key( @@ -4005,100 +4060,114 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()> { - self.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - #[cfg(test)] - if record.owner_account_id == "owner-account" { - // Unrelated store tests use this explicit fixture identity. Keep the - // production owner contract strict while giving those fixtures a real - // User Account row instead of reviving ownerless Workspace setup. - tx.execute( - "INSERT OR IGNORE INTO accounts ( - account_id, kind, handle, display_name, created_at, updated_at - ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", - params![record.owner_account_id.as_str(), record.created_at.as_str()], - )?; - } - let owner_kind = tx - .query_row( - "SELECT kind FROM accounts WHERE account_id = ?1", - params![record.owner_account_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if owner_kind.as_deref() != Some("user") { - return Err(Error::Store(format!( - "Workspace owner `{}` must reference an existing User Account", - record.owner_account_id - ))); - } - let current_owner = tx - .query_row( - "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", - params![record.workspace_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if current_owner - .as_deref() - .is_some_and(|owner| owner != record.owner_account_id) - { - return Err(Error::Store(format!( - "Workspace `{}` owner is immutable through upsert", - record.workspace_id - ))); - } - tx.execute( - r#"INSERT INTO workspaces ( - workspace_id, owner_account_id, display_name, state, created_at, updated_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(workspace_id) DO UPDATE SET - display_name = excluded.display_name, - state = excluded.state, - updated_at = excluded.updated_at"#, - params![ - record.workspace_id, - record.owner_account_id, - record.display_name, - record.state, - record.created_at, - record.updated_at, - ], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_memory_settings ( - workspace_id, language, created_at, updated_at - ) VALUES (?1, 'English', ?2, ?3)"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_signing_identities ( - workspace_id, key_id, algorithm, public_key, public_key_fingerprint, - private_material_ref, state, created_at, provisioned_at, updated_at - ) VALUES ( - ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, - 'workspace-signing/' || ?1 || '/ed25519-v1', - 'pending_provisioning', ?2, NULL, ?3 - )"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.commit()?; - Ok(()) - })?; - self.materialize_workspace_config(&record.workspace_id, &record.created_at) + let store = self.clone(); + let record = record.clone(); + self.db_executor + .run(move || { + let store = &store; + let record = &record; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + #[cfg(test)] + if record.owner_account_id == "owner-account" { + // Unrelated store tests use this explicit fixture identity. Keep the + // production owner contract strict while giving those fixtures a real + // User Account row instead of reviving ownerless Workspace setup. + tx.execute( + "INSERT OR IGNORE INTO accounts ( + account_id, kind, handle, display_name, created_at, updated_at + ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", + params![record.owner_account_id.as_str(), record.created_at.as_str()], + )?; + } + let owner_kind = tx + .query_row( + "SELECT kind FROM accounts WHERE account_id = ?1", + params![record.owner_account_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if owner_kind.as_deref() != Some("user") { + return Err(Error::Store(format!( + "Workspace owner `{}` must reference an existing User Account", + record.owner_account_id + ))); + } + let current_owner = tx + .query_row( + "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", + params![record.workspace_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if current_owner + .as_deref() + .is_some_and(|owner| owner != record.owner_account_id) + { + return Err(Error::Store(format!( + "Workspace `{}` owner is immutable through upsert", + record.workspace_id + ))); + } + tx.execute( + r#"INSERT INTO workspaces ( + workspace_id, owner_account_id, display_name, state, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(workspace_id) DO UPDATE SET + display_name = excluded.display_name, + state = excluded.state, + updated_at = excluded.updated_at"#, + params![ + record.workspace_id, + record.owner_account_id, + record.display_name, + record.state, + record.created_at, + record.updated_at, + ], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_memory_settings ( + workspace_id, language, created_at, updated_at + ) VALUES (?1, 'English', ?2, ?3)"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_signing_identities ( + workspace_id, key_id, algorithm, public_key, public_key_fingerprint, + private_material_ref, state, created_at, provisioned_at, updated_at + ) VALUES ( + ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, + 'workspace-signing/' || ?1 || '/ed25519-v1', + 'pending_provisioning', ?2, NULL, ?3 + )"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.commit()?; + Ok(()) + })?; + store.materialize_workspace_config(&record.workspace_id, &record.created_at) + }) + .await } async fn get_workspace(&self, workspace_id: &str) -> Result> { - self.with_conn(|conn| { - conn.query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional() - .map_err(Error::from) - }) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor.run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + store.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional() + .map_err(Error::from) + }) + }).await } async fn update_workspace_display_name( @@ -4107,58 +4176,68 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_updated_at: &str, display_name: &str, ) -> Result> { - validate_identifier("workspace_id", workspace_id)?; - validate_non_empty("expected_updated_at", expected_updated_at)?; - validate_non_empty("display_name", display_name)?; - self.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let current = tx - .query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional()?; - let Some(current) = current else { - tx.commit()?; - return Ok(None); - }; - if current.updated_at != expected_updated_at { - tx.commit()?; - return Ok(None); - } - if current.display_name == display_name { - tx.commit()?; - return Ok(Some(current)); - } + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let expected_updated_at = expected_updated_at.to_owned(); + let display_name = display_name.to_owned(); + self.db_executor.run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let expected_updated_at = expected_updated_at.as_str(); + let display_name = display_name.as_str(); + validate_identifier("workspace_id", workspace_id)?; + validate_non_empty("expected_updated_at", expected_updated_at)?; + validate_non_empty("display_name", display_name)?; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current = tx + .query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional()?; + let Some(current) = current else { + tx.commit()?; + return Ok(None); + }; + if current.updated_at != expected_updated_at { + tx.commit()?; + return Ok(None); + } + if current.display_name == display_name { + tx.commit()?; + return Ok(Some(current)); + } - let now = chrono::Utc::now(); - let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) - .ok() - .map(|previous| previous.with_timezone(&chrono::Utc)) - .filter(|previous| *previous >= now) - .map(|previous| previous + chrono::Duration::nanoseconds(1)) - .unwrap_or(now) - .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); - let changed = tx.execute( - r#"UPDATE workspaces - SET display_name = ?3, updated_at = ?4 - WHERE workspace_id = ?1 AND updated_at = ?2"#, - params![workspace_id, expected_updated_at, display_name, updated_at], - )?; - if changed != 1 { + let now = chrono::Utc::now(); + let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) + .ok() + .map(|previous| previous.with_timezone(&chrono::Utc)) + .filter(|previous| *previous >= now) + .map(|previous| previous + chrono::Duration::nanoseconds(1)) + .unwrap_or(now) + .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + let changed = tx.execute( + r#"UPDATE workspaces + SET display_name = ?3, updated_at = ?4 + WHERE workspace_id = ?1 AND updated_at = ?2"#, + params![workspace_id, expected_updated_at, display_name, updated_at], + )?; + if changed != 1 { + tx.commit()?; + return Ok(None); + } + let updated = WorkspaceRecord { + display_name: display_name.to_string(), + updated_at, + ..current + }; tx.commit()?; - return Ok(None); - } - let updated = WorkspaceRecord { - display_name: display_name.to_string(), - updated_at, - ..current - }; - tx.commit()?; - Ok(Some(updated)) - }) + Ok(Some(updated)) + }) + }).await } fn create_workspace_bootstrap( @@ -4678,7 +4757,17 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result> { - SqliteWorkspaceStore::get_workspace_runtime_binding(self, workspace_id, runtime_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::get_workspace_runtime_binding(store, workspace_id, runtime_id) + }) + .await } async fn list_workspace_runtime_bindings( @@ -4686,7 +4775,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, include_revoked: bool, ) -> Result> { - SqliteWorkspaceStore::list_workspace_runtime_bindings(self, workspace_id, include_revoked) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + SqliteWorkspaceStore::list_workspace_runtime_bindings( + store, + workspace_id, + include_revoked, + ) + }) + .await } async fn has_other_active_workspace_runtime_binding( @@ -4694,11 +4795,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( - self, - workspace_id, - runtime_id, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + ) + }) + .await } async fn delete_workspace_runtime_binding( @@ -4706,7 +4817,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - SqliteWorkspaceStore::delete_workspace_runtime_binding(self, workspace_id, runtime_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::delete_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + ) + }) + .await } async fn reserve_runtime_removal( @@ -4718,22 +4843,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: &str, config_digest: &str, ) -> Result { - SqliteWorkspaceStore::reserve_runtime_removal( - self, - workspace_id, - runtime_id, - operation_id, - request_fingerprint, - expected_binding_id, - config_digest, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let operation_id = operation_id.to_owned(); + let request_fingerprint = request_fingerprint.to_owned(); + let expected_binding_id = expected_binding_id.to_owned(); + let config_digest = config_digest.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let operation_id = operation_id.as_str(); + let request_fingerprint = request_fingerprint.as_str(); + let expected_binding_id = expected_binding_id.as_str(); + let config_digest = config_digest.as_str(); + SqliteWorkspaceStore::reserve_runtime_removal( + store, + workspace_id, + runtime_id, + operation_id, + request_fingerprint, + expected_binding_id, + config_digest, + ) + }) + .await } async fn get_runtime_removal( &self, operation_id: &str, ) -> Result> { - SqliteWorkspaceStore::get_runtime_removal(self, operation_id) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::get_runtime_removal(store, operation_id) + }) + .await } async fn mark_runtime_removal_failed( @@ -4741,14 +4892,36 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, failure_category: &str, ) -> Result { - SqliteWorkspaceStore::mark_runtime_removal_failed(self, operation_id, failure_category) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + let failure_category = failure_category.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + let failure_category = failure_category.as_str(); + SqliteWorkspaceStore::mark_runtime_removal_failed( + store, + operation_id, + failure_category, + ) + }) + .await } async fn commit_runtime_binding_removal( &self, operation_id: &str, ) -> Result { - SqliteWorkspaceStore::commit_runtime_binding_removal(self, operation_id) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::commit_runtime_binding_removal(store, operation_id) + }) + .await } async fn complete_runtime_removal( @@ -4756,18 +4929,34 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, runtime_registration_removed: bool, ) -> Result { - SqliteWorkspaceStore::complete_runtime_removal( - self, - operation_id, - runtime_registration_removed, - ) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::complete_runtime_removal( + store, + operation_id, + runtime_registration_removed, + ) + }) + .await } async fn list_resumable_runtime_removals( &self, workspace_id: &str, ) -> Result> { - SqliteWorkspaceStore::list_resumable_runtime_removals(self, workspace_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + SqliteWorkspaceStore::list_resumable_runtime_removals(store, workspace_id) + }) + .await } async fn upsert_workspace_runtime_binding_record( @@ -4775,7 +4964,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record: WorkspaceRuntimeBinding, replace: bool, ) -> Result { - SqliteWorkspaceStore::upsert_workspace_runtime_binding(self, record, replace) + let store = self.clone(); + + self.db_executor + .run(move || { + let store = &store; + SqliteWorkspaceStore::upsert_workspace_runtime_binding(store, record, replace) + }) + .await } async fn revoke_workspace_runtime_binding_record( @@ -4784,12 +4980,24 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, revoked_at: &str, ) -> Result { - SqliteWorkspaceStore::revoke_workspace_runtime_binding( - self, - workspace_id, - runtime_id, - revoked_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let revoked_at = revoked_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let revoked_at = revoked_at.as_str(); + SqliteWorkspaceStore::revoke_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + revoked_at, + ) + }) + .await } async fn put_workspace_runtime_binding_key( @@ -4798,12 +5006,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: Option<&str>, actor_account_id: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - SqliteWorkspaceStore::put_workspace_runtime_binding_key( - self, - record, - expected_binding_id, - actor_account_id, - ) + let store = self.clone(); + let expected_binding_id = expected_binding_id.map(str::to_owned); + let actor_account_id = actor_account_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let expected_binding_id = expected_binding_id.as_deref(); + let actor_account_id = actor_account_id.as_str(); + SqliteWorkspaceStore::put_workspace_runtime_binding_key( + store, + record, + expected_binding_id, + actor_account_id, + ) + }) + .await } async fn update_workspace_runtime_binding_metadata( @@ -4814,14 +5032,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore { base_url: &str, updated_at: &str, ) -> Result { - SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( - self, - workspace_id, - runtime_id, - display_name, - base_url, - updated_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let display_name = display_name.to_owned(); + let base_url = base_url.to_owned(); + let updated_at = updated_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let display_name = display_name.as_str(); + let base_url = base_url.as_str(); + let updated_at = updated_at.as_str(); + SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( + store, + workspace_id, + runtime_id, + display_name, + base_url, + updated_at, + ) + }) + .await } async fn revoke_workspace_runtime_binding_key( @@ -4832,14 +5066,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore { actor_account_id: &str, revoked_at: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( - self, - workspace_id, - runtime_id, - expected_binding_id, - actor_account_id, - revoked_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let expected_binding_id = expected_binding_id.to_owned(); + let actor_account_id = actor_account_id.to_owned(); + let revoked_at = revoked_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let expected_binding_id = expected_binding_id.as_str(); + let actor_account_id = actor_account_id.as_str(); + let revoked_at = revoked_at.as_str(); + SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( + store, + workspace_id, + runtime_id, + expected_binding_id, + actor_account_id, + revoked_at, + ) + }) + .await } async fn list_workspace_runtime_binding_audit( @@ -4848,12 +5098,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, limit: usize, ) -> Result> { - SqliteWorkspaceStore::list_workspace_runtime_binding_audit( - self, - workspace_id, - runtime_id, - limit, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::list_workspace_runtime_binding_audit( + store, + workspace_id, + runtime_id, + limit, + ) + }) + .await } async fn consume_worker_mutation_source_jti( @@ -4865,21 +5125,35 @@ impl ControlPlaneStore for SqliteWorkspaceStore { now_seconds: u64, consumed_at: &str, ) -> Result { - self.with_conn(|conn| { - let transaction = conn.unchecked_transaction()?; - transaction.execute( - "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", - params![now_seconds], - )?; - let inserted = transaction.execute( - r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( - workspace_id, runtime_id, jti, expires_at, consumed_at - ) VALUES (?1, ?2, ?3, ?4, ?5)"#, - params![workspace_id, runtime_id, jti, expires_at, consumed_at], - )?; - transaction.commit()?; - Ok(inserted == 1) - }) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let jti = jti.to_owned(); + let consumed_at = consumed_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let jti = jti.as_str(); + let consumed_at = consumed_at.as_str(); + store.with_conn(|conn| { + let transaction = conn.unchecked_transaction()?; + transaction.execute( + "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", + params![now_seconds], + )?; + let inserted = transaction.execute( + r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( + workspace_id, runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, ?2, ?3, ?4, ?5)"#, + params![workspace_id, runtime_id, jti, expires_at, consumed_at], + )?; + transaction.commit()?; + Ok(inserted == 1) + }) + }) + .await } fn plan_worker_removal( @@ -14686,7 +14960,7 @@ mod tests { fn drive_grants_migrate_and_preserve_audit_through_revoke_worker_delete_and_restart() { let t = tempfile::tempdir().unwrap(); let path = t.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let legacy = Connection::open_in_memory().unwrap(); configure_sqlite(&legacy).unwrap(); legacy.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY); @@ -14776,7 +15050,7 @@ mod tests { Some(other) ); drop(store); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( store .get_workspace_drive_grant("drive-ws", &first.grant_id) @@ -14811,7 +15085,7 @@ mod tests { } #[test] fn drive_grants_baseline_constraints_and_worker_lifetime_fail_closed() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let proposed = drive_grant_fixture(&store); let first = store.create_workspace_drive_grant(&proposed).unwrap(); store.with_conn(|c| { @@ -14847,7 +15121,7 @@ mod tests { fn schema_v74_migrates_historical_worker_projection_rows_without_counter_state() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("workspace.db"); - drop(SqliteWorkspaceStore::open(&path).unwrap()); + drop(SqliteWorkspaceStore::open_for_test(&path).unwrap()); let conn = Connection::open(&path).unwrap(); configure_sqlite(&conn).unwrap(); conn.execute_batch( @@ -15172,7 +15446,7 @@ mod tests { #[tokio::test] async fn external_workdir_grant_persists_typed_source_and_fences_generations() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .upsert_account(&AccountRecord { account_id: "owner".to_string(), @@ -15544,7 +15818,7 @@ mod tests { fn runtime_removal_test_store() -> (tempfile::TempDir, SqliteWorkspaceStore) { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -15928,7 +16202,7 @@ mod tests { let path = temp.path().join("server.db"); prepare_retained_schema(&path, 59); - let migrated = SqliteWorkspaceStore::open(&path).unwrap(); + let migrated = SqliteWorkspaceStore::open_for_test(&path).unwrap(); migrated .with_conn(|conn| { let table_count: i64 = conn.query_row( @@ -15964,7 +16238,7 @@ mod tests { let path = temp.path().join("server.db"); prepare_retained_schema(&path, 60); - drop(SqliteWorkspaceStore::open(&path).unwrap()); + drop(SqliteWorkspaceStore::open_for_test(&path).unwrap()); let conn = Connection::open(&path).unwrap(); assert_eq!( current_schema_version(&conn).unwrap(), @@ -15994,7 +16268,7 @@ mod tests { #[test] fn schema_v59_runtime_removal_migration_rolls_back_partial_ddl_and_marker() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap(); store .with_conn(|conn| { let trigger_names = { @@ -16258,7 +16532,7 @@ mod tests { fn fresh_database_uses_only_the_canonical_workspace_baseline() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { let migrations = conn @@ -16406,7 +16680,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .collect::>(), (83..=LATEST_SCHEMA_VERSION).collect::>() ); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let assignment = store .get_active_ticket_worker_assignment("workspace-a", "ticket") .unwrap() @@ -16877,7 +17151,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let applied = SqliteWorkspaceStore::migrate_database(&path).unwrap(); assert_eq!(applied, plan); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -17426,7 +17700,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "migrated databases must reject incomplete singleton leases" ); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let legacy = RuntimeWorkerRef::new("legacy-runtime", "legacy-orchestrator"); let lease = store .require_current_worker_singleton_owner("workspace-a", &legacy) @@ -17614,7 +17888,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -17715,7 +17989,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(conn); - let error = SqliteWorkspaceStore::open(&path) + let error = SqliteWorkspaceStore::open_for_test(&path) .err() .expect("migration must fail") .to_string(); @@ -17740,7 +18014,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx fn runtime_binding_identity_and_trust_uniqueness_are_workspace_scoped() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -17842,7 +18116,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert!( reopened .list_workspace_runtime_bindings("workspace-a", false) @@ -17892,7 +18166,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx fn workspace_runtime_connection_persists_without_authentication_evidence() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -17937,7 +18211,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); assert!(store.workspace_runtime_binding_matches(&persisted).unwrap()); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( reopened .get_workspace_runtime_binding("workspace-a", "runtime-a") @@ -18191,7 +18465,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn runtime_binding_key_mutations_are_identity_guarded_idempotent_and_audited() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18329,7 +18603,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn embedded_runtime_binding_can_explicitly_rotate_restart_identity() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18429,7 +18703,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx conn.execute_batch("DROP TABLE typed_ticket_artifacts") .unwrap(); - let result = SqliteWorkspaceStore::from_connection(conn); + let result = SqliteWorkspaceStore::from_connection_for_test(conn); let error = match result { Ok(_) => panic!("schema drift unexpectedly passed startup verification"), Err(error) => error, @@ -18441,7 +18715,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn workspace_schema_rejects_cross_workspace_ticket_repository_reference_at_write_time() { let dir = tempfile::tempdir().unwrap(); let database_path = dir.path().join("workspace.sqlite"); - let store = SqliteWorkspaceStore::open(&database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); for workspace_id in ["workspace-a", "workspace-b"] { store .upsert_workspace(&WorkspaceRecord { @@ -18495,14 +18769,14 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(backend); - SqliteWorkspaceStore::open(&database_path).unwrap(); + SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); } #[tokio::test] async fn migrates_sqlite_and_preserves_workspace_record() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("control-plane.sqlite"); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); assert!( @@ -18520,7 +18794,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx }; store.upsert_workspace(&record).await.unwrap(); - let reopened = SqliteWorkspaceStore::open(&db).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!( reopened.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION @@ -18535,7 +18809,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn workspace_display_name_update_is_timestamp_guarded_and_preserves_identity() { let dir = tempfile::tempdir().unwrap(); let database_path = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); let record = WorkspaceRecord { workspace_id: "workspace-a".to_string(), owner_account_id: "owner-account".to_string(), @@ -18574,7 +18848,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx updated ); drop(store); - let reopened = SqliteWorkspaceStore::open(database_path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(database_path).unwrap(); assert_eq!( reopened .get_workspace(&record.workspace_id) @@ -18588,7 +18862,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn objective_creation_allocates_and_resolves_workspace_resource_key() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".into(), @@ -18634,7 +18908,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn worker_create_reservation_allocates_uuid_before_runtime_and_replays_exact_input() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -18938,7 +19212,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn worker_singleton_restart_retry_replays_exact_workdir_intent_and_rejects_changes() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -18998,7 +19272,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&database).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap(); let retried = reopened .reserve_worker_create( "workspace-a", @@ -19042,7 +19316,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn worker_singleton_ownership_is_workspace_scoped_durable_and_fences_old_generations() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); for workspace_id in ["workspace-a", "workspace-b"] { store .upsert_workspace(&WorkspaceRecord { @@ -19155,7 +19429,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); - let reopened = SqliteWorkspaceStore::open(database).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap(); assert_eq!( reopened .current_worker_singleton_owner("workspace-a", key) @@ -19168,7 +19442,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn concurrent_worker_singleton_reservations_have_one_winner() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19188,7 +19462,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let database = database.clone(); let barrier = barrier.clone(); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open(database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(database).unwrap(); let memory = store.get_workspace_memory_settings("workspace-a").unwrap(); barrier.wait(); store.reserve_worker_create( @@ -19208,7 +19482,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .collect::>(); assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); assert_eq!(results.iter().filter(|result| result.is_err()).count(), 1); - let store = SqliteWorkspaceStore::open(database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(database).unwrap(); let owner = store .current_worker_singleton_owner("workspace-a", "shared-key") .unwrap() @@ -19224,7 +19498,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn workspace_flow_sources_pin_content_and_builtins_stay_resources() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19317,7 +19591,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "returning to earlier content must reuse the immutable content" ); drop(store); - let reopened = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); assert_eq!( reopened .get_flow_source_content( @@ -19573,7 +19847,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx } ); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "case-workspace").unwrap(); store .with_conn(|conn| { @@ -19740,7 +20014,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); drop(backend); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); reopened .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -19807,7 +20081,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx before ); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "case-workspace").unwrap(); assert_eq!( backend @@ -19941,8 +20215,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("equivalence.db"); prepare_retained_schema(&path, 90); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); for object in [ "workspace_tags", "typed_ticket_tags", @@ -19995,7 +20269,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn schema_v91_case_constraint_verification_rejects_legacy_view_predicates() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.with_conn(|conn| { conn.execute_batch("DROP VIEW ticket_active_worker_assignments; CREATE VIEW ticket_active_worker_assignments AS @@ -20156,7 +20430,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx // boundary under test. Targets need not be materialized by a Runtime. async fn manual_worker_start_store() -> (tempfile::TempDir, SqliteWorkspaceStore) { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -20244,7 +20518,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let threads = (0..2) .map(|index| { let store = - SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")) + .unwrap(); let barrier = barrier.clone(); std::thread::spawn(move || { barrier.wait(); @@ -20609,7 +20884,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .retain_ticket_claim_recovery("w", "claim", &fingerprint, &changed) .is_err() ); - let reopened = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + let reopened = + SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); assert_eq!( reopened.get_ticket_claim_recovery("w", "claim").unwrap(), Some(recovery) @@ -20667,7 +20943,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx store .rollback_ticket_assignment_operation("w", "claim") .unwrap(); - let reopened = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + let reopened = + SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); let receipt = reopened .get_ticket_assignment_operation("w", "claim") .unwrap() @@ -20887,7 +21164,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx use ticket::TicketBackend; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -21015,7 +21292,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn ticket_worker_assignment_replaces_current_and_preserves_audit_history() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -21204,7 +21481,7 @@ INSERT INTO worker_registry ( ) .unwrap(); drop(store); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); let pending = store .get_ticket_assignment_operation("workspace-a", "reserved-operation") .unwrap() @@ -21328,7 +21605,7 @@ INSERT INTO worker_registry ( async fn role_assignment_routes_and_manual_start_are_state_fenced_atomically() { let dir = tempfile::tempdir().unwrap(); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-role".to_string(), @@ -21617,7 +21894,7 @@ INSERT INTO worker_registry ( fn startup_rejects_missing_workspace_resource_constraint_trigger() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch("DROP TRIGGER ticket_worker_assignments_validate_insert")?; @@ -21626,7 +21903,7 @@ INSERT INTO worker_registry ( .unwrap(); drop(store); - let error = match SqliteWorkspaceStore::open(&path) { + let error = match SqliteWorkspaceStore::open_for_test(&path) { Ok(_) => panic!("missing assignment constraint trigger must fail closed"), Err(error) => error, }; @@ -21642,7 +21919,7 @@ INSERT INTO worker_registry ( fn startup_accepts_retained_assignment_history_after_parent_cleanup() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -21706,7 +21983,7 @@ INSERT INTO worker_registry ( .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); reopened .with_conn(|conn| { let retained: i64 = conn.query_row( @@ -21734,7 +22011,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn workspace_bootstrap_validates_key_and_rejects_duplicate_workspace() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .upsert_account(&AccountRecord { account_id: "owner-account".to_string(), @@ -21866,7 +22143,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn repository_records_round_trip() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -21947,7 +22224,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22022,7 +22299,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn legacy_memory_reset_is_atomic_scoped_idempotent_and_exact() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); for (workspace_id, display_name) in [ ("local-dev", "Local Dev"), ("other-workspace", "Other Workspace"), @@ -22189,7 +22466,7 @@ INSERT INTO worker_registry ( async fn worker_workdir_registry_round_trips_and_preserves_pinned_retention() { let temp = tempfile::tempdir().unwrap(); let db = temp.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: "owner-account".to_string(), @@ -22558,7 +22835,7 @@ INSERT INTO worker_registry ( ); } drop(store); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!( store .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 0, None) @@ -22588,7 +22865,7 @@ INSERT INTO worker_registry ( let workdir_conflict = store.attach_worker_workdir(&workdir_conflict).unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&db).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!( reopened .list_worker_workdir_links("local-dev", &worker.worker) @@ -22606,7 +22883,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn worker_projection_preserves_equal_local_ids_from_distinct_runtimes() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22732,7 +23009,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn worker_projection_reconciles_snapshots_and_preserves_unavailable_lifecycle() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22907,7 +23184,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn late_runtime_event_does_not_recreate_retention_deleted_catalog_worker() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -23002,7 +23279,7 @@ INSERT INTO worker_registry ( async fn worker_control_grants_are_idempotent_scoped_and_support_internal_invalidation() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("control-grants.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-control".to_string(), @@ -23072,7 +23349,7 @@ INSERT INTO worker_registry ( ); drop(store); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); assert_eq!( store .list_active_worker_control_grants( @@ -23130,7 +23407,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn account_and_login_records_round_trip() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { @@ -23489,9 +23766,9 @@ INSERT INTO worker_registry ( async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("resource-jobs.sqlite"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); seed_resource_job_workspace(&store).await; - let other = SqliteWorkspaceStore::open(&path).unwrap(); + let other = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let requests = [ resource_job_request("first", "subject-a", "candidate-1"), resource_job_request("followup", "subject-a", "candidate-2"), @@ -23607,7 +23884,7 @@ INSERT INTO worker_registry ( async fn backend_job_resource_retry_and_unknown_are_fenced_by_durable_cleanup() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("cleanup-jobs.sqlite"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); seed_resource_job_workspace(&store).await; let request = resource_job_request("failed", "subject-a", "candidate-1"); let first = store @@ -23713,7 +23990,7 @@ INSERT INTO worker_registry ( .unwrap(); finish_resource_job_cleanup(&store, &second); drop(store); - let restarted = SqliteWorkspaceStore::open(&path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let active = restarted .find_active_backend_job_for_resource( "resource-workspace", @@ -23773,7 +24050,7 @@ INSERT INTO worker_registry ( async fn backend_job_resource_retry_races_new_intent_atomically_and_remains_bounded() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("retry-race.sqlite"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); seed_resource_job_workspace(&store).await; // Generic resources use the same common reservation API without domain grants. let mut request = resource_job_request("generic", "subject-a", "candidate-1"); @@ -23796,7 +24073,7 @@ INSERT INTO worker_registry ( let mut followup = request.clone(); followup.job_id = "generic-followup".into(); followup.input = serde_json::json!({"candidate": "new"}); - let other = SqliteWorkspaceStore::open(&path).unwrap(); + let other = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let barrier = std::sync::Barrier::new(2); let (retry, reserve) = std::thread::scope(|scope| { let a = scope.spawn(|| { @@ -23882,7 +24159,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_all_older_attempt_cleanups_fence_retry_and_resource_release() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); seed_resource_job_workspace(&store).await; let mut request = resource_job_request("legacy-overlap", "subject-a", "candidate-1"); request.limits.max_attempts = 3; @@ -24006,7 +24283,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_empty_batch_surface_refresh_locks_subject_and_defers_candidates() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); seed_resource_job_workspace(&store).await; for surface_state in ["ungenerated", "stale", "failed"] { let subject = format!("subject-{surface_state}"); @@ -24126,7 +24403,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_completed_resource_releases_only_after_worker_cleanup() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); seed_resource_job_workspace(&store).await; let request = resource_job_request("completed", "subject-a", "candidate-1"); let first = store @@ -24180,7 +24457,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_jobs_fence_results_project_ownership_and_bound_retries() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let workspace_id = "workspace-job"; store .upsert_account(&AccountRecord { diff --git a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs index 0ea9c64e7..69190eb1f 100644 --- a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs +++ b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs @@ -263,7 +263,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64 let tags_before = retained.as_ref().map(|_| classification_rows(&conn)); let auth_before = fixture_auth_rows(&conn); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?,93); assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11); @@ -490,7 +490,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history let metadata_before = stored_fixture_rows(&conn, "typed_ticket_case_metadata"); let auth_before = fixture_auth_rows(&conn); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?,93); diff --git a/crates/workspace-server/src/workdir_create_operations.rs b/crates/workspace-server/src/workdir_create_operations.rs index a41fc8ca7..a98284b1e 100644 --- a/crates/workspace-server/src/workdir_create_operations.rs +++ b/crates/workspace-server/src/workdir_create_operations.rs @@ -750,7 +750,7 @@ mod tests { #[test] fn retry_keeps_resolved_config_and_credential_candidates_after_fallback_moves() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord { workspace_id: "workspace".to_string(), owner_account_id: "owner-account".to_string(), diff --git a/crates/workspace-server/src/workdir_removal.rs b/crates/workspace-server/src/workdir_removal.rs index 9c1f2a327..13b481fb4 100644 --- a/crates/workspace-server/src/workdir_removal.rs +++ b/crates/workspace-server/src/workdir_removal.rs @@ -925,7 +925,7 @@ mod tests { } async fn seeded_store() -> (SqliteWorkspaceStore, WorkdirRegistryRecord) { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .upsert_account(&AccountRecord { account_id: "account-a".to_string(), diff --git a/crates/workspace-server/src/worker_projection.rs b/crates/workspace-server/src/worker_projection.rs index ed910de04..fc8140132 100644 --- a/crates/workspace-server/src/worker_projection.rs +++ b/crates/workspace-server/src/worker_projection.rs @@ -5,7 +5,7 @@ use chrono::Utc; use protocol::subscription::{ EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot, }; -use tokio::sync::broadcast; +use tokio::sync::{Semaphore, broadcast}; use tokio::task::JoinHandle; use crate::runtime_subscription::{ @@ -59,7 +59,14 @@ pub struct WorkerProjectionService { events: broadcast::Sender, ordering: Mutex<()>, attach_gate: tokio::sync::Mutex<()>, - tasks: Mutex>>, + tasks: Mutex>, +} + +struct RuntimeProjectionLifetime { + consumer: Option>, + // The permit belongs to the actual DB closure, not its cancellable async waiter. + // Keep this owner in the map even if attachment initialization is cancelled. + operations: Arc, } impl WorkerProjectionService { @@ -75,17 +82,89 @@ impl WorkerProjectionService { }) } + /// Stops consumers but does not claim that accepted database work has finished. + /// Async shutdown callers must use `shutdown_async` to drain accepted operations. pub fn shutdown(&self) { - for (_, task) in self + for lifetime in self .tasks .lock() .expect("Worker projection task lock poisoned") - .drain() + .values() { - task.abort(); + if let Some(task) = &lifetime.consumer { + task.abort(); + } } } + pub async fn shutdown_async(&self) { + let _attach = self.attach_gate.lock().await; + self.shutdown(); + let runtime_ids = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .keys() + .cloned() + .collect::>(); + for runtime_id in runtime_ids { + self.drain_runtime(&runtime_id).await; + } + self.tasks + .lock() + .expect("Worker projection task lock poisoned") + .clear(); + } + + async fn drain_runtime(&self, runtime_id: &str) { + let previous = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .get_mut(runtime_id) + .map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone())); + if let Some((consumer, operations)) = previous { + if let Some(consumer) = consumer { + consumer.abort(); + let _ = consumer.await; + } + // Aborting the waiter cannot abort a queued/running synchronous job. + // Cancellation here leaves the gate in the map for the next drain. + let _drained = operations + .acquire_owned() + .await + .expect("operation gate closed"); + } + } + + // Caller holds attach_gate. Install ownership before any initialization await. + async fn replace_runtime(&self, runtime_id: &str) -> Arc { + self.drain_runtime(runtime_id).await; + let operations = Arc::new(Semaphore::new(1)); + self.tasks + .lock() + .expect("Worker projection task lock poisoned") + .insert( + runtime_id.to_string(), + RuntimeProjectionLifetime { + consumer: None, + operations: operations.clone(), + }, + ); + operations + } + + pub async fn subscribe_ordered_async( + self: &Arc, + limit: usize, + ) -> crate::Result { + let service = self.clone(); + self.store + .db_executor() + .run(move || service.subscribe_ordered(limit)) + .await + } + pub fn subscribe_ordered(&self, limit: usize) -> crate::Result { let _ordering = self .ordering @@ -125,19 +204,10 @@ impl WorkerProjectionService { broker: &RuntimeSubscriptionBroker, runtime_id: &str, ) -> Result<(), RuntimeSubscriptionBrokerError> { - // Serialize replacement and wait for the old consumer to stop before the new - // lifetime can apply its snapshot. Late notifications can therefore never - // cross a Runtime ownership boundary. + // Serialize replacement, stop admission by aborting the consumer, and then + // drain ownership held by the actual SQL jobs before the new snapshot. let _attach = self.attach_gate.lock().await; - let previous = self - .tasks - .lock() - .expect("Worker projection task lock poisoned") - .remove(runtime_id); - if let Some(previous) = previous { - previous.abort(); - let _ = previous.await; - } + let operations = self.replace_runtime(runtime_id).await; let mut subscription = broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?; @@ -148,12 +218,25 @@ impl WorkerProjectionService { break; }; let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. }); - if let Err(error) = self.apply_broker_event(runtime_id, event) { + if let Err(error) = self + .apply_broker_event_owned_async(runtime_id, event, operations.clone()) + .await + { tracing::warn!( runtime_id, error = %error, "failed to initialize durable Worker projection; restarting from a fresh snapshot" ); + if matches!( + error, + crate::Error::DbOperationOutcomeUnknown + | crate::Error::DbExecutorUnavailable + | crate::Error::DbCapacityExceeded + ) { + // Do not automatically replay a possibly committed mutation or + // spin on executor overload; attachment must be retried explicitly. + return Err(RuntimeSubscriptionBrokerError::Closed); + } subscription.restart()?; continue; } @@ -169,12 +252,27 @@ impl WorkerProjectionService { let Some(service) = service.upgrade() else { return; }; - if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) { + if let Err(error) = service + .apply_broker_event_owned_async( + task_runtime_id.as_str(), + event, + operations.clone(), + ) + .await + { tracing::warn!( runtime_id = %task_runtime_id, error = %error, "failed to update durable Worker projection; restarting from a fresh snapshot" ); + if matches!( + error, + crate::Error::DbOperationOutcomeUnknown + | crate::Error::DbExecutorUnavailable + | crate::Error::DbCapacityExceeded + ) { + return; + } if subscription.restart().is_err() { return; } @@ -184,10 +282,61 @@ impl WorkerProjectionService { self.tasks .lock() .expect("Worker projection task lock poisoned") - .insert(runtime_id, task); + .get_mut(&runtime_id) + .expect("Runtime lifetime installed before initialization") + .consumer = Some(task); Ok(()) } + #[cfg(test)] + async fn seed_observation_async( + self: &Arc, + runtime_id: &str, + worker: &protocol::subscription::SubscriptionWorker, + observed_at: &str, + ) -> crate::Result<()> { + let service = self.clone(); + let runtime_id = runtime_id.to_string(); + let worker = worker.clone(); + let observed_at = observed_at.to_string(); + self.store + .db_executor() + .run(move || service.seed_observation(&runtime_id, &worker, &observed_at)) + .await + } + + pub async fn refresh_async(self: &Arc, worker: &RuntimeWorkerRef) -> crate::Result<()> { + let service = self.clone(); + let worker = worker.clone(); + self.store + .db_executor() + .run(move || service.refresh(&worker)) + .await + } + + async fn apply_broker_event_owned_async( + self: &Arc, + runtime_id: &str, + event: BrokerSubscriptionEvent, + operations: Arc, + ) -> crate::Result<()> { + let operation = operations + .acquire_owned() + .await + .expect("operation gate closed"); + let service = self.clone(); + let runtime_id = runtime_id.to_string(); + self.store + .db_executor() + .run(move || { + // The permit must outlive SQL commit AND ordered publication, even + // when the executor result receiver has been dropped by cancellation. + let _operation = operation; + service.apply_broker_event(&runtime_id, event) + }) + .await + } + pub fn seed_observation( &self, runtime_id: &str, @@ -313,7 +462,11 @@ mod tests { tempfile::TempDir, ) { let temp = tempfile::tempdir().unwrap(); - let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap()); + let store = Arc::new( + SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")) + .unwrap() + .with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()), + ); store .upsert_account(&AccountRecord { account_id: "owner".to_string(), @@ -355,6 +508,266 @@ mod tests { } } + fn observation( + state: protocol::subscription::SubscriptionWorkerState, + ) -> protocol::subscription::SubscriptionWorker { + protocol::subscription::SubscriptionWorker { + restore_observation_token: None, + worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(), + runtime_id: Some("runtime-a".to_string()), + resource_key: None, + availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, + worker_state: None, + state, + has_running_internal_workers: false, + workspace_id: Some(WORKSPACE_ID.to_string()), + display_name: None, + profile: None, + job: None, + workdir_attachments: Vec::new(), + } + } + + // Real second-connection contention; the controller releases automatically on + // failure, so a regression that blocks the Tokio thread cannot hang the suite. + struct WriteLock { + release: Option>, + controller: Option>, + } + + impl WriteLock { + fn acquire(path: std::path::PathBuf) -> Self { + let (release, released) = std::sync::mpsc::channel(); + let (locked, ready) = std::sync::mpsc::channel(); + let controller = std::thread::spawn(move || { + let connection = rusqlite::Connection::open(path).unwrap(); + connection.execute_batch("BEGIN IMMEDIATE").unwrap(); + locked.send(()).unwrap(); + let _ = released.recv_timeout(std::time::Duration::from_secs(3)); + connection.execute_batch("ROLLBACK").unwrap(); + }); + ready.recv().unwrap(); + Self { + release: Some(release), + controller: Some(controller), + } + } + + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + } + } + + impl Drop for WriteLock { + fn drop(&mut self) { + self.release(); + if let Some(controller) = self.controller.take() { + controller.join().unwrap(); + } + } + } + + async fn wait_for_sql(service: &WorkerProjectionService) { + tokio::time::timeout(std::time::Duration::from_secs(1), async { + // With the second connection holding the write lock, a held ordering + // gate identifies the accepted synchronous mutation, not a fake future. + while service.ordering.try_lock().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("projection job did not enter synchronous SQL"); + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload() + { + use protocol::subscription::SubscriptionWorkerState; + let (store, service, temp) = fixture().await; + store + .upsert_worker_registry(&worker("runtime-a", "worker-a")) + .unwrap(); + let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); + assert_eq!(subscription.take_snapshot().len(), 1); + let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); + let publishing_service = service.clone(); + let publication = tokio::spawn(async move { + publishing_service + .seed_observation_async( + "runtime-a", + &observation(SubscriptionWorkerState::Idle), + "2", + ) + .await + }); + wait_for_sql(&service).await; + let started = std::time::Instant::now(); + let (control, controlled) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + control.send("control progressed").unwrap(); + }); + let (_, control_result) = tokio::join!( + tokio::time::sleep(std::time::Duration::from_millis(20)), + controlled, + ); + assert_eq!(control_result.unwrap(), "control progressed"); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "SQLite blocked the current-thread executor" + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + lock.release(); + publication.await.unwrap().unwrap(); + let committed = subscription.recv().await.unwrap(); + let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); + let snapshot = snapshot.take_snapshot(); + assert_eq!( + committed.changes, + vec![WorkerCatalogChange::Upsert(snapshot[0].clone())] + ); + assert_eq!( + snapshot[0].observation.as_ref().unwrap().worker.state, + SubscriptionWorkerState::Idle + ); + service + .seed_observation_async( + "runtime-a", + &observation(SubscriptionWorkerState::Running), + "3", + ) + .await + .unwrap(); + // Delayed readers receive the first transaction's payload, not a re-query + // of mutable authority after a later commit. + let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else { + panic!("expected upsert"); + }; + assert_eq!( + first.observation.as_ref().unwrap().worker.state, + SubscriptionWorkerState::Idle + ); + service + .refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a")) + .await + .unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() { + use crate::runtime_subscription::BrokerSubscriptionEvent; + use protocol::subscription::SubscriptionWorkerState; + let (store, service, temp) = fixture().await; + store + .upsert_worker_registry(&worker("runtime-a", "worker-a")) + .unwrap(); + let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); + subscription.take_snapshot(); + let operations = service.replace_runtime("runtime-a").await; + let old_operations = operations.clone(); + let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); + let old_service = service.clone(); + let consumer = tokio::spawn(async move { + old_service + .apply_broker_event_owned_async( + "runtime-a", + BrokerSubscriptionEvent::Event { + payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted { + worker: observation(SubscriptionWorkerState::Running), + }, + }, + operations, + ) + .await + .unwrap(); + }); + service + .tasks + .lock() + .unwrap() + .get_mut("runtime-a") + .unwrap() + .consumer = Some(consumer); + wait_for_sql(&service).await; + let replacing_service = service.clone(); + let mut replacement = tokio::spawn(async move { + let _attach = replacing_service.attach_gate.lock().await; + replacing_service.replace_runtime("runtime-a").await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) + .await + .is_err() + ); + replacement.abort(); + assert!(replacement.await.unwrap_err().is_cancelled()); + assert_eq!( + old_operations.available_permits(), + 0, + "cancelled drain lost accepted SQL ownership" + ); + // A second replacement must still observe the old gate after cancellation. + let replacing_service = service.clone(); + let mut replacement = tokio::spawn(async move { + let _attach = replacing_service.attach_gate.lock().await; + let operations = replacing_service.replace_runtime("runtime-a").await; + replacing_service + .apply_broker_event_owned_async( + "runtime-a", + BrokerSubscriptionEvent::Snapshot { + snapshot: protocol::subscription::SubscriptionSnapshot::Workers { + workers: vec![observation(SubscriptionWorkerState::Idle)], + }, + }, + operations, + ) + .await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) + .await + .is_err() + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + lock.release(); + replacement.await.unwrap().unwrap(); + assert_eq!(old_operations.available_permits(), 1); + let old_commit = subscription.recv().await.unwrap(); + let new_commit = subscription.recv().await.unwrap(); + for (commit, expected) in [ + (old_commit, SubscriptionWorkerState::Running), + (new_commit, SubscriptionWorkerState::Idle), + ] { + let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else { + panic!("expected one committed upsert"); + }; + assert_eq!(record.observation.as_ref().unwrap().worker.state, expected); + } + let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); + assert_eq!( + snapshot.take_snapshot()[0] + .observation + .as_ref() + .unwrap() + .worker + .state, + SubscriptionWorkerState::Idle + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + service.shutdown_async().await; + assert!(service.tasks.lock().unwrap().is_empty()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn snapshot_registration_is_atomic_with_concurrent_publication() { let (store, service, _temp) = fixture().await; diff --git a/crates/workspace-server/src/worker_source.rs b/crates/workspace-server/src/worker_source.rs index 3d51476d1..3edbb5d3e 100644 --- a/crates/workspace-server/src/worker_source.rs +++ b/crates/workspace-server/src/worker_source.rs @@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError { Replay, #[error("source Worker is not a current member of this Workspace Runtime catalog")] WorkerCatalogMembership, + #[error("source proof authority is unavailable")] + Unavailable, #[error("source proof authority failed: {0}")] Authority(String), } +impl From for WorkerMutationSourceProofError { + fn from(error: crate::Error) -> Self { + match error { + crate::Error::DbCapacityExceeded + | crate::Error::DbExecutorUnavailable + | crate::Error::DbOperationOutcomeUnknown + | crate::Error::Sqlite(_) => Self::Unavailable, + other => Self::Authority(other.to_string()), + } + } +} + pub async fn verify_worker_remove_source( api: &WorkspaceApi, proof: PresentedWorkerMutationSourceProof<'_>, @@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with( let trusted = store .get_workspace_runtime_binding(&config.workspace_id, &unverified.iss) .await - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))? + .map_err(WorkerMutationSourceProofError::from)? .filter(|record| record.revoked_at.is_none()) .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?; let public_key = trusted.public_key.as_str(); @@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with( runtime_id: claims.iss.clone(), worker_id: claims.worker_id.clone(), }; - let member = store - .get_worker_registry(&config.workspace_id, &worker) - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; - if member.is_none() { - return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); - } + let workspace_id = config.workspace_id.clone(); + let catalog_store = Arc::clone(store); store - .require_current_worker_singleton_owner(&config.workspace_id, &worker) - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + .db_executor() + .run(move || { + let member = catalog_store + .get_worker_registry(&workspace_id, &worker) + .map_err(WorkerMutationSourceProofError::from)?; + if member.is_none() { + return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); + } + catalog_store + .require_current_worker_singleton_owner(&workspace_id, &worker) + .map_err(WorkerMutationSourceProofError::from)?; + Ok(()) + }) + .await?; let consumed_at = chrono::Utc::now().to_rfc3339(); let consumed = store @@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with( &consumed_at, ) .await - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + .map_err(WorkerMutationSourceProofError::from)?; if !consumed { return Err(WorkerMutationSourceProofError::Replay); } diff --git a/crates/workspace-server/src/worker_source/runtime_request.rs b/crates/workspace-server/src/worker_source/runtime_request.rs index 9e79fd3b1..b8c2e7226 100644 --- a/crates/workspace-server/src/worker_source/runtime_request.rs +++ b/crates/workspace-server/src/worker_source/runtime_request.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use worker_runtime::auth::{ RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims, RuntimeRequestSourceExpectation, decode_runtime_request_source_claims, @@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError { pub(crate) exp: Option, pub(crate) now: Option, pub(crate) time_delta_seconds: Option, + /// Operational unavailability is not a proof or principal rejection. + pub(crate) availability: bool, } impl RuntimeRequestProofError { + pub fn is_availability_error(&self) -> bool { + self.availability + } + + fn unavailable(mut self, reason: &'static str) -> Self { + self.reason = reason; + self.availability = true; + self + } + fn undecoded() -> Self { Self { reason: "invalid", @@ -47,6 +61,7 @@ impl RuntimeRequestProofError { exp: None, now: None, time_delta_seconds: None, + availability: false, } } @@ -106,6 +121,18 @@ impl RuntimeRequestProofError { } } +impl From for RuntimeRequestProofError { + fn from(error: crate::Error) -> Self { + let reason = match error { + crate::Error::DbCapacityExceeded => "db_capacity_exceeded", + crate::Error::DbExecutorUnavailable => "db_executor_unavailable", + crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown", + _ => "database_authority_unavailable", + }; + Self::undecoded().unavailable(reason) + } +} + /// Domain-separated fixed-length fingerprints, not raw request-derived IDs. /// Operators can correlate a known ID by applying this same function. pub fn diagnostic_id_hash(id: &str) -> String { @@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof( body_digest: &str, ) -> Result { verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + Arc::clone(&api.store), &api.config, proof, workspace_id, @@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof( #[allow(clippy::too_many_arguments)] pub async fn verify_runtime_request_source_proof_with_store( - store: &dyn ControlPlaneStore, + store: Arc, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store( #[allow(clippy::too_many_arguments)] pub(crate) async fn verify_runtime_request_source_proof_with_clock( - store: &dyn ControlPlaneStore, + store: Arc, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( let unverified = decode_runtime_request_source_claims(proof) .map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?; let diagnostic = RuntimeRequestProofError::decoded(&unverified); - let audience = remote_audience(config, workspace_id) - .map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?; + let audience = remote_audience(config, workspace_id).map_err(|_| { + diagnostic + .clone() + .unavailable("backend_audience_unavailable") + })?; let trusted = store .get_workspace_runtime_binding(workspace_id, &unverified.iss) .await .map_err(|_| { diagnostic .clone() - .reason("runtime_trust_authority_unavailable") + .unavailable("runtime_trust_authority_unavailable") })? .filter(|record| record.revoked_at.is_none()) .ok_or_else(|| { @@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( &consumed_at, ) .await - .map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))? + .map_err(|_| { + diagnostic + .clone() + .unavailable("replay_authority_unavailable") + })? { return Err(diagnostic.reason("replay")); } @@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( runtime_id: claims.iss.clone(), worker_id: worker_id.to_owned(), }; - let member = store - .get_worker_registry(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_catalog_authority_unavailable") - })?; - let reserved = store - .has_active_worker_create_reservation(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_reservation_authority_unavailable") - })?; - if member.is_none() && !reserved { - return Err(diagnostic.reason("worker_catalog_membership")); - } - store - .require_current_worker_singleton_owner(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_singleton_authority_denied") + let workspace_id = workspace_id.to_owned(); + let executor = store.db_executor(); + let stage_diagnostic = diagnostic.clone(); + // Only synchronous catalog authority lives in this closure. The trust + // and JTI async wrappers above already offload through the executor. + executor + .run(move || { + let member = store + .get_worker_registry(&workspace_id, &worker) + .map_err(|_| { + stage_diagnostic + .clone() + .unavailable("worker_catalog_authority_unavailable") + })?; + let reserved = store + .has_active_worker_create_reservation(&workspace_id, &worker) + .map_err(|_| { + stage_diagnostic + .clone() + .unavailable("worker_reservation_authority_unavailable") + })?; + if member.is_none() && !reserved { + return Err(stage_diagnostic.reason("worker_catalog_membership")); + } + store + .require_current_worker_singleton_owner(&workspace_id, &worker) + .map_err(|error| { + if matches!( + error, + crate::Error::DbCapacityExceeded + | crate::Error::DbExecutorUnavailable + | crate::Error::DbOperationOutcomeUnknown + | crate::Error::Sqlite(_) + ) { + stage_diagnostic + .clone() + .unavailable("worker_singleton_authority_unavailable") + } else { + stage_diagnostic + .clone() + .reason("worker_singleton_authority_denied") + } + })?; + Ok(()) + }) + .await + .map_err(|error: RuntimeRequestProofError| { + if error.verification == "unverified" && error.is_availability_error() { + diagnostic.clone().unavailable(error.reason) + } else { + error + } })?; } Ok(VerifiedRuntimeRequestSource { @@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( authentication_binding_id: trusted.binding_id, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::{ + AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode, + WorkspaceRuntimeBinding, + }; + use std::time::Duration; + use worker_runtime::auth::{ + BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner, + request_body_digest, + }; + + const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa"; + const NOW: i64 = 1_700_000_000; + const PATH: &str = "/api/runtime/v1/resources/fetch"; + + // rusqlite's busy handler accepts a function pointer, not a closure. Each + // fixture owns a dedicated single-worker executor, so this thread-local + // handshake is isolated from all parallel tests and product connections. + thread_local! { + static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell, + std::sync::mpsc::Receiver<()>, + )>> = const { std::cell::RefCell::new(None) }; + } + + fn acknowledge_sqlite_contention(_attempt: i32) -> bool { + SQLITE_BUSY_HANDSHAKE.with(|slot| { + let Some((blocked, retry)) = slot.borrow_mut().take() else { + return false; + }; + let _ = blocked.send(()); + // The test releases SQLite's writer lock before allowing retry. + // Dropping the sender on panic also releases this worker promptly. + retry.recv_timeout(Duration::from_secs(3)).is_ok() + }) + } + + async fn fixture( + root: &std::path::Path, + ) -> (Arc, ServerConfig, String, String) { + let db_path = root.join("auth.db"); + let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::open_for_test(&db_path) + .unwrap() + .with_db_executor(executor), + ); + store + .upsert_account(&AccountRecord { + account_id: "account-auth".into(), + kind: "user".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + let workspace = WorkspaceRecord { + workspace_id: WORKSPACE.into(), + owner_account_id: "account-auth".into(), + display_name: "Auth".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }; + store.upsert_workspace(&workspace).await.unwrap(); + let config = ServerConfig { + workspace_id: WORKSPACE.into(), + workspace_display_name: "Auth".into(), + workspace_created_at: "1".into(), + workspace_execution_root: root.to_owned(), + database_path: db_path, + data_root: root.to_owned(), + embedded_runtime_store_root: root.join("runtime"), + static_assets_dir: None, + auth: crate::server::AuthConfig::Passkey { + rp_id: "localhost".into(), + origin: "http://localhost".into(), + public_base_url: "http://localhost".into(), + cookie_name: "session".into(), + }, + max_records: 10, + repositories: vec![], + runtime_event_sources: vec![], + remote_runtime_sources: vec![], + backend_base_url: Some("server-auth".into()), + }; + let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap(); + store + .upsert_workspace_runtime_binding_record( + WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: "runtime-auth".into(), + display_name: "Auth".into(), + base_url: "http://127.0.0.1:1".into(), + public_key: identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer, + created_at: "1".into(), + updated_at: "1".into(), + revoked_at: None, + }, + false, + ) + .await + .unwrap(); + let binding = ControlPlaneStore::get_workspace_runtime_binding( + store.as_ref(), + WORKSPACE, + "runtime-auth", + ) + .await + .unwrap() + .unwrap(); + let proof = RuntimeRequestSourceSigner::from_identity(&identity) + .issue( + "server-auth", + WORKSPACE, + None, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + b"{}", + NOW, + 30, + ) + .unwrap(); + (store, config, proof, binding.binding_id) + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() { + let root = tempfile::tempdir().unwrap(); + let (store, config, proof, binding_id) = fixture(root.path()).await; + // A real independent SQLite writer prevents JTI insertion while trust + // reads remain possible. Rollback on Drop also releases this on panic. + let mut writer = rusqlite::Connection::open(&config.database_path).unwrap(); + let lock = writer + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .unwrap(); + let (blocked, contention_reached) = tokio::sync::oneshot::channel(); + let (retry, retry_rx) = std::sync::mpsc::channel(); + let setup_store = store.clone(); + store + .db_executor() + .run(move || -> crate::Result<()> { + SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx))); + setup_store.with_conn(|conn| { + conn.busy_handler(Some(acknowledge_sqlite_contention))?; + Ok(()) + }) + }) + .await + .unwrap(); + let request_store = store.clone(); + let request_config = config.clone(); + let request_proof = proof.clone(); + let task = tokio::spawn(async move { + verify_runtime_request_source_proof_with_clock( + request_store, + &request_config, + &request_proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(1), contention_reached) + .await + .expect( + "SQLite JTI persistence must report real writer contention without blocking Tokio", + ) + .unwrap(); + // The busy callback acknowledges an actual SQLite write blocked by the + // independent writer, then holds retry until explicitly released. + assert!( + !task.is_finished(), + "JTI must remain pending while SQLite writer is locked" + ); + let control = tokio::spawn(async { "control-responsive" }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), control) + .await + .unwrap() + .unwrap(), + "control-responsive" + ); + // This timer is the behavior being tested, not a guessed wait for SQL. + tokio::time::timeout( + Duration::from_secs(1), + tokio::time::sleep(Duration::from_millis(1)), + ) + .await + .unwrap(); + assert!( + !task.is_finished(), + "timer must advance while JTI remains blocked" + ); + lock.rollback().unwrap(); + retry.send(()).unwrap(); + let source = tokio::time::timeout(Duration::from_secs(3), task) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(source.runtime_id, "runtime-auth"); + assert_eq!(source.authentication_binding_id, binding_id); + let replay = verify_runtime_request_source_proof_with_clock( + store, + &config, + &proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ) + .await + .unwrap_err(); + assert_eq!(replay.reason, "replay"); + assert!(!replay.is_availability_error()); + } + + #[tokio::test] + async fn saturated_database_is_availability_failure_not_invalid_proof() { + let root = tempfile::tempdir().unwrap(); + let (store, config, proof, _) = fixture(root.path()).await; + // Fill the isolated executor with a blocked job and queued jobs. No + // scheduling sleeps: the worker explicitly acknowledges its start. + let executor = store.db_executor(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let blocker_executor = executor.clone(); + let blocker = tokio::spawn(async move { + blocker_executor + .run(move || -> crate::Result<()> { + let _ = started.send(()); + release_rx.recv().unwrap(); + Ok(()) + }) + .await + }); + tokio::time::timeout(Duration::from_secs(1), started_rx) + .await + .unwrap() + .unwrap(); + let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) })); + assert!(futures::poll!(queued.as_mut()).is_pending()); + let outcome = tokio::time::timeout( + Duration::from_secs(1), + verify_runtime_request_source_proof_with_clock( + store, + &config, + &proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ), + ) + .await; + // Release before assertions so a failed contract never strands a worker. + release.send(()).unwrap(); + blocker.await.unwrap().unwrap(); + queued.await.unwrap(); + let error = outcome + .expect("capacity rejection must not wait for queued SQL") + .unwrap_err(); + assert_eq!(error.reason, "runtime_trust_authority_unavailable"); + assert!(error.is_availability_error()); + } +} diff --git a/crates/workspace-server/src/workspace_catalog.rs b/crates/workspace-server/src/workspace_catalog.rs index 5d9679d9f..78ce46f45 100644 --- a/crates/workspace-server/src/workspace_catalog.rs +++ b/crates/workspace-server/src/workspace_catalog.rs @@ -285,7 +285,7 @@ mod tests { #[tokio::test] async fn create_is_atomic_and_exact_retries_converge() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let service = WorkspaceCatalogService::new( store.clone(), Arc::new( @@ -339,7 +339,7 @@ mod tests { #[test] fn create_rejects_noncanonical_repository_keys_without_persisting_a_workspace() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let service = WorkspaceCatalogService::new( store.clone(), Arc::new( @@ -378,7 +378,7 @@ mod tests { for with_repository in [false, true] { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let owner_account_id = owner_account(store.as_ref()); let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default()); let service = WorkspaceCatalogService::new( @@ -418,7 +418,7 @@ mod tests { drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let restarted = WorkspaceCatalogService::new(store.clone(), materials); let created = restarted.create(request, owner_account_id).unwrap(); let identity = store @@ -435,7 +435,7 @@ mod tests { for with_repository in [false, true] { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let owner_account_id = owner_account(store.as_ref()); let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default()); let service = WorkspaceCatalogService::new(store.clone(), materials.clone()); @@ -486,7 +486,7 @@ mod tests { drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let restarted = WorkspaceCatalogService::new(store.clone(), materials); let created = restarted.create(request, owner_account_id).unwrap(); let identity = store @@ -500,7 +500,7 @@ mod tests { #[tokio::test] async fn idempotency_key_reuse_with_different_payload_is_rejected() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner_account_id = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store, @@ -532,7 +532,7 @@ mod tests { #[test] fn repository_free_create_initializes_authority_and_concurrent_retries_converge() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), @@ -611,7 +611,7 @@ mod tests { #[test] fn replay_reads_current_repository_at_initial_key_and_never_recreates_it() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), @@ -673,13 +673,13 @@ mod tests { repository: None, }; let (owner, created) = { - let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new(store, materials.clone()); let created = service.create(request.clone(), owner.clone()).unwrap(); (owner, created) }; - let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); let reopened = WorkspaceCatalogService::new(store.clone(), materials); let replay = reopened.create(request, owner).unwrap(); assert_eq!(replay.workspace, created.workspace); @@ -739,7 +739,7 @@ mod tests { #[test] fn create_rejects_non_user_account_owners() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); store .upsert_account(&AccountRecord { account_id: "organization-owner".to_string(), @@ -777,7 +777,7 @@ mod tests { #[test] fn catalog_list_is_scoped_to_the_required_owner_account() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner_a = owner_account(store.as_ref()); let owner_b = "account-owner-b".to_string(); store @@ -845,7 +845,7 @@ mod tests { #[test] fn remote_repository_creation_persists_typed_source_without_auth_metadata() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner_account_id = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index e241f8aee..5003d4153 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -236,7 +236,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { "Workspace deletion request exceeds bounded field limits".to_string(), )); } - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { if let Some(existing) = read_operation(tx, &request.operation_id)? { if existing.actor_account_id != actor_account_id { return Err(Error::WorkspacePermissionDenied( @@ -369,7 +369,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { child_operation_id: &str, ) -> Result { validate_operation_id(child_operation_id)?; - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { let operation = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? .response; @@ -411,7 +411,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { failure_category: Option<&str>, ) -> Result { validate_operation_projection(child_operation_ids, blockers)?; - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { let current = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? .response; @@ -462,7 +462,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { &self, operation_id: &str, ) -> Result { - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?; let operation = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; @@ -922,7 +922,7 @@ mod tests { use tempfile::tempdir; fn setup() -> (SqliteWorkspaceStore, String, String) { - seed_workspaces(SqliteWorkspaceStore::in_memory().expect("store")) + seed_workspaces(SqliteWorkspaceStore::in_memory_for_test().expect("store")) } fn seed_workspaces(store: SqliteWorkspaceStore) -> (SqliteWorkspaceStore, String, String) { @@ -989,7 +989,7 @@ mod tests { let dir = tempdir().expect("tempdir"); let path = dir.path().join("server.db"); let (store, owner, workspace_id) = - seed_workspaces(SqliteWorkspaceStore::open(&path).expect("store")); + seed_workspaces(SqliteWorkspaceStore::open_for_test(&path).expect("store")); for workspace in ["workspace-a", "workspace-b"] { let backend = ticket::SqliteTicketBackend::open_verified(&path, workspace).unwrap(); let initial = backend @@ -1019,7 +1019,7 @@ mod tests { // Seed domain handling evidence under a transaction. This fixture // exercises owner purge, not singleton authentication or delivery. store - .with_transaction(|tx| { + .with_immediate_transaction(|tx| { ticket::sqlite_update_ticket_assignment_tracking_in_transaction( tx, workspace, diff --git a/crates/workspace-server/src/workspace_drive_host.rs b/crates/workspace-server/src/workspace_drive_host.rs index 410c0c049..bc0bfc390 100644 --- a/crates/workspace-server/src/workspace_drive_host.rs +++ b/crates/workspace-server/src/workspace_drive_host.rs @@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost { impl WorkspaceDriveHost { pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result { let storage = storage.clone(); - let registration = tokio::task::spawn_blocking(move || Drive::register(&storage)) - .await - .map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))? - .map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?; + let registration = crate::db_executor::DbExecutor::shared() + .run(move || { + Drive::register(&storage).map_err(|error| { + Error::Store(format!("failed to register Drive storage: {error}")) + }) + }) + .await?; Ok(Self { registration, purged: AtomicBool::new(false), @@ -80,21 +83,21 @@ impl WorkspaceDriveHost { } let storage = storage.clone(); let registration = self.registration.clone(); - let drive = tokio::task::spawn_blocking(move || { - reject_symlink_ancestors(&root)?; - // Domain IMMEDIATE transactions lock the attached server authority - // as well as Drive metadata. Deletion reservation and cached-handle - // operations therefore serialize with other server DB writes. - Drive::open_with_workspace_authority( - &storage, - ®istration, - &root, - &authority_database, - ) - .map_err(Error::from) - }) - .await - .map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??; + let drive = crate::db_executor::DbExecutor::shared() + .run(move || { + reject_symlink_ancestors(&root)?; + // Domain IMMEDIATE transactions lock the attached server authority + // as well as Drive metadata. Deletion reservation and cached-handle + // operations therefore serialize with other server DB writes. + Drive::open_with_workspace_authority( + &storage, + ®istration, + &root, + &authority_database, + ) + .map_err(Error::from) + }) + .await?; *cached = Some(drive.clone()); Ok(drive) } @@ -109,9 +112,9 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root, authority_database).await?; - tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from)) + crate::db_executor::DbExecutor::shared() + .run(move || drive.fence().map_err(Error::from)) .await - .map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))? } pub(crate) async fn purge( @@ -124,20 +127,20 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root.clone(), authority_database).await?; - tokio::task::spawn_blocking(move || -> Result<()> { - reject_symlink_ancestors(&root)?; - // Domain purge persists its tombstone before removing blobs/nodes/receipts. - drive.purge()?; - // Remove the empty blob root, never the data root or another Workspace. - // Missing roots are successful retries; other failures retain FeatureStorage. - match std::fs::remove_dir_all(&root) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - } - }) - .await - .map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??; + crate::db_executor::DbExecutor::shared() + .run(move || -> Result<()> { + reject_symlink_ancestors(&root)?; + // Domain purge persists its tombstone before removing blobs/nodes/receipts. + drive.purge()?; + // Remove the empty blob root, never the data root or another Workspace. + // Missing roots are successful retries; other failures retain FeatureStorage. + match std::fs::remove_dir_all(&root) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + }) + .await?; // A later signing-material/finalization failure can retry after the // feature DB has been closed and removed by FeatureStorage.delete. self.purged.store(true, Ordering::Release); diff --git a/crates/workspace-server/src/workspace_signing_identity.rs b/crates/workspace-server/src/workspace_signing_identity.rs index f724f2ca1..959f28805 100644 --- a/crates/workspace-server/src/workspace_signing_identity.rs +++ b/crates/workspace-server/src/workspace_signing_identity.rs @@ -780,7 +780,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); store .upsert_account(&AccountRecord { account_id: "account-1".to_string(), @@ -833,7 +833,7 @@ mod tests { drop(failing_service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let service = WorkspaceSigningIdentityService::new(store.clone(), materials.clone()); let provisioned = service .provision_existing("workspace-1", "account-1") @@ -962,7 +962,7 @@ mod tests { ); drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let restarted = WorkspaceSigningIdentityService::new(store.clone(), materials.clone()); let recovered = restarted .provision_existing("workspace-2", "account-1") diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index 8b0e0b818..a62faa34e 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription( worker_id, runtime_id: Some(runtime_id), } => { - let connection = match WorkspaceWorker::resolve( + let connection = match WorkspaceWorker::resolve_async( &api, &runtime_id, worker_id.as_str(), - ) { + ) + .await + { Ok(worker) => worker .connect_protocol(&context) .await @@ -366,7 +368,8 @@ async fn run_workspace_workers( ) { let Ok(mut projection) = api .worker_projection - .subscribe_ordered(api.config.max_records.max(1)) + .subscribe_ordered_async(api.config.max_records.max(1)) + .await else { let _ = send_rejected( &outbound, @@ -377,11 +380,22 @@ async fn run_workspace_workers( .await; return; }; - let mut workers = projection - .take_snapshot() - .into_iter() - .filter_map(|record| project_registry_worker(&api, record)) - .collect::>(); + let Ok(mut workers) = project_registry_workers_async( + api.store.clone(), + api.config.workspace_id.clone(), + projection.take_snapshot(), + ) + .await + else { + let _ = send_rejected( + &outbound, + request_id, + SubscriptionRejectionCode::Internal, + "failed to enrich durable Worker projection".to_string(), + ) + .await; + return; + }; sort_workers(&mut workers); if send_frame( &outbound, @@ -406,7 +420,29 @@ async fn run_workspace_workers( for change in event.changes { let payload = match change { crate::worker_projection::WorkerProjectionChange::Upsert(record) => { - let Some(worker) = project_registry_worker(&api, record) else { + let workers = project_registry_workers_async( + api.store.clone(), + api.config.workspace_id.clone(), + vec![record], + ) + .await; + let Ok(mut workers) = workers else { + // Admission failure cannot silently lose a live + // update: terminate and require a fresh snapshot. + let _ = send_frame( + &outbound, + SubscriptionFrame::new(SubscriptionFramePayload::Event( + SubscriptionEvent::SubscriptionClosed { + subscription_id: subscription_id.clone(), + code: SubscriptionTerminationCode::Lagged, + message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(), + }, + )), + ) + .await; + return; + }; + let Some(worker) = workers.pop() else { continue; }; SubscriptionEventPayload::WorkerUpserted { worker } @@ -450,8 +486,28 @@ async fn run_workspace_workers( } } +async fn project_registry_workers_async( + store: std::sync::Arc, + workspace_id: String, + records: Vec, +) -> crate::Result> { + store + .db_executor() + .run(move || { + // Keep the committed Worker payload; only the existing Workspace-scoped + // workdir/history/repository enrichment reads occur inside this job. + Ok(records + .into_iter() + .filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record)) + .collect()) + }) + .await +} + +// Synchronous helpers stay synchronous; they run only inside the job above. fn project_registry_worker( - api: &WorkspaceApi, + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, record: crate::store::WorkerRegistryProjectionRecord, ) -> Option { let runtime_id = record.registry.worker.runtime_id.clone(); @@ -491,29 +547,25 @@ fn project_registry_worker( attempt_id: job.attempt_id, purpose: job.purpose, }); - if !project_working_directory(api, &record.registry.worker, &mut worker) { + if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) { return None; } Some(worker) } fn project_working_directory( - api: &WorkspaceApi, + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, worker_ref: &RuntimeWorkerRef, worker: &mut SubscriptionWorker, ) -> bool { - let Ok(links) = api - .store - .list_worker_workdir_links(&api.config.workspace_id, worker_ref) - else { + let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else { return false; }; if !links.is_empty() { let mut attachments = Vec::with_capacity(links.len()); for link in links { - let Ok(Some(workdir)) = api - .store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id) else { return false; }; @@ -533,9 +585,8 @@ fn project_working_directory( } worker.workdir_attachments = attachments; } else { - let Ok(has_link_history) = api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref) + let Ok(has_link_history) = + store.worker_workdir_link_history_exists(workspace_id, worker_ref) else { return false; }; @@ -543,13 +594,17 @@ fn project_working_directory( worker.workdir_attachments.clear(); } } - project_repository_keys(api, worker) + project_repository_keys(store, workspace_id, worker) } -fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool { +fn project_repository_keys( + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, + worker: &mut SubscriptionWorker, +) -> bool { project_repository_keys_with(worker, |repository_id| { - api.store - .get_repository(&api.config.workspace_id, repository_id) + store + .get_repository(workspace_id, repository_id) .ok() .flatten() .map(|repository| repository.repository_key) @@ -634,9 +689,8 @@ mod tests { protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap() } - #[test] - fn external_workdir_attachment_does_not_remove_workspace_worker() { - let mut worker = SubscriptionWorker { + fn external_worker() -> SubscriptionWorker { + SubscriptionWorker { restore_observation_token: None, worker_id: worker_id(), runtime_id: Some("runtime-1".to_string()), @@ -660,8 +714,12 @@ mod tests { .unwrap(), }, ], - }; + } + } + #[test] + fn external_workdir_attachment_does_not_remove_workspace_worker() { + let mut worker = external_worker(); assert!(project_repository_keys_with(&mut worker, |_| { panic!("External Workdir must not require Repository projection") })); @@ -673,6 +731,110 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository() + { + use crate::store::{ + AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord, + WorkspaceRecord, + }; + let store = std::sync::Arc::new( + SqliteWorkspaceStore::in_memory_for_test() + .unwrap() + .with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()), + ); + store + .upsert_account(&AccountRecord { + account_id: "owner".into(), + kind: "user".into(), + handle: "owner".into(), + display_name: "Owner".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "workspace-1".into(), + owner_account_id: "owner".into(), + display_name: "Enrichment".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .await + .unwrap(); + let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1"); + store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: "workspace-1".into(), + worker: worker_ref.clone(), + display_name: "Durable name".into(), + profile: Some("builtin:coder".into()), + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2") + .unwrap(); + let committed = store + .worker_registry_projection("workspace-1", &worker_ref) + .unwrap() + .unwrap(); + // Change authority after capturing the payload, as can happen while async + // enrichment is queued. It must not re-query and replace the committed Worker. + let mut later = external_worker(); + later.state = protocol::subscription::SubscriptionWorkerState::Running; + store + .apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3") + .unwrap(); + let projected = project_registry_workers_async( + store.clone(), + "workspace-1".into(), + vec![committed.clone()], + ) + .await + .unwrap(); + assert_eq!(projected.len(), 1); + let worker = &projected[0]; + assert_eq!( + worker.state, + protocol::subscription::SubscriptionWorkerState::Idle + ); + assert_eq!( + worker.availability, + protocol::subscription::SubscriptionWorkerAvailability::Observed + ); + assert_eq!(worker.display_name.as_deref(), Some("Durable name")); + assert_eq!(worker.profile.as_deref(), Some("builtin:coder")); + assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1")); + assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1")); + assert_eq!( + worker.workdir_attachments, + external_worker().workdir_attachments + ); + let mut unresolved = committed; + unresolved + .observation + .as_mut() + .unwrap() + .worker + .workdir_attachments[0] + .repository_id = Some("missing-repository".into()); + assert!( + project_registry_workers_async(store, "workspace-1".into(), vec![unresolved]) + .await + .unwrap() + .is_empty() + ); + } + async fn run_with_events(events: mpsc::Receiver) -> SubscriptionResponse { let (control_outbound, mut control_receiver) = mpsc::channel(1); let (protocol_outbound, _protocol_receiver) = mpsc::channel(1); diff --git a/docs/development/backend-db-execution.md b/docs/development/backend-db-execution.md new file mode 100644 index 000000000..8dd00b478 --- /dev/null +++ b/docs/development/backend-db-execution.md @@ -0,0 +1,115 @@ +# Backend bounded synchronous execution + +## Ownership and admission + +`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner. +Use `executor.run(move || synchronous_operation(...)).await` with owned input and +services. The closure owns the **whole** synchronous operation, including connection +acquisition, mutex wait, the existing SQL transaction, commit, and ordered local +publication. Do not split source authorization, receipts, CAS, audit, or saved +results around an await. Do not change connection ownership or introduce a read pool. + +Production stores share one DB worker and a queue of 32 jobs. Admission uses a +nonblocking queue send and a separate actual-job budget (worker count plus queue +capacity); rejection means the operation was **not accepted**. HTTP maps capacity +and executor unavailability to 503, not authentication failure or deletion conflict. +Tests can inject a dedicated executor with explicit limits, without process-global +settings or environment variables. Ordinary SQLite fixtures use `open_for_test`, +`in_memory_for_test`, or `from_connection_for_test`: their clones share their own +1-worker/32-queued owner, not the production singleton or another fixture's drain. +Server/Workspace fixture constructors likewise inject a lazy `HostExecutors` owner; +all child Workspaces and weak retention adapters use that same owner. Host and +cleanup still have the production 2-worker/16-queued limits. Dropping the final +sender closes its queue after accepted work finishes; idle fixture threads are +not kept in a process-global fixture cache. + +The restore intent journal and Workspace deletion use +`with_immediate_transaction`: read authority/journal state only after acquiring +SQLite write ownership. A DEFERRED read-to-write upgrade can return BUSY without +invoking SQLite's busy handler when another connection is writing. Moving these +whole helpers to an offloaded caller must not create an automatic mutation retry. + + +Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter: + +- does not interrupt, resubmit, or retry SQL; +- does not return its queue/job slot before execution and discarded-result cleanup; +- does not release the request lifecycle lease before actual completion. + +Synchronous post-commit continuations inherit the admitted lifecycle source when +submitting another bounded job. This is not implicit inheritance by detached Tokio +tasks. An operation panic has an unknown outcome even if SQL already committed; +return a structured unknown-outcome error and diagnostic, never retry the mutation. +A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all +jobs accepted before its barrier is first polled and waits for actual cleanup; it +is not a queue close and does not include later admissions. Lifecycle callers must +stop their producers/admission before destroying resources. + +## Runtime/network boundaries + +A synchronous Runtime workflow must not monopolize the sole DB worker while +waiting for callbacks that themselves need DB authority. The Server's separate +finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job +ownership mechanism, not an unbounded `spawn_blocking` queue. + +Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can +use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an +owned workflow needs async resource locks, `run_host_workflow` polls it on a Host +worker using the original Tokio handle; do not build a replacement runtime for +cached services and do not submit to and await the Host lane from inside itself. +Control/session guards needed by an accepted mutation belong to that actual job. + +Backend Job cleanup has a third finite lane (two workers, 16 queued jobs), so a +long Run teardown wait cannot starve ordinary Host control/stop requests. It is +admitted before its durable cleanup claim. Rejection leaves Pending/Failed +ownership recoverable instead of abandoning an Executing claim. Ticket +checker dispatch and timeout processing use bounded admission; overload leaves the +saved reservation/deadline diagnosable. They are not automatically retried after an +unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof +of model processing. + +Remote subscription and Workdir authorization are async callbacks: their complete +live binding check and fresh request signing are offloaded together. A queue error +must not fall back to a configured bearer token. Synchronous CLI/migration callers +and existing synchronous Runtime adapters retain their explicit blocking boundary. + +## Caller recipe (including assignment APIs) + +```rust +let store = api.store.clone(); +let workspace_id = api.workspace_id().to_owned(); +let worker = worker.clone(); +let record = api.store.db_executor().run(move || { + store.get_worker_registry(&workspace_id, &worker) +}).await?; +``` + +This is an illustration, not a new authority API. Assignment delivery scan, +reserve, prepare, acknowledgement and start authorization must keep their existing +whole transaction helpers. Additional assignment diagnostics, answer mutations, +and Ticket queries use the same owned caller boundary; do not add a second ledger +or reconstruct processing evidence from Worker liveness. + +## Regression evidence + +- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread + timer/control task progresses; accepted/cancelled queue bounds; rollback; panic + before/after commit; discarded reply cleanup and retained authority; child jobs. +- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope + opening/mutation under a separate SQLite connection's write lock; overload; + cancelled HTTP jobs and Workspace deletion drain; same-budget fixture isolation, + saturated/cancelled Host jobs versus cleanup and paused Tokio time; restore intent + contention before the first authority read, with one persisted tuple and no retry. + The terminal checker fixture keeps its original Tokio driver alive through accepted cleanup, + then shuts down projection producers and drains actual DB jobs. +- `worker_projection::tests`: real SQL contention, commit/payload publication, + ordered snapshot registration, and replacement draining a cancelled consumer's + accepted SQL before applying a new Runtime snapshot. +- Auth/proof and Runtime subscription tests protect availability classification, + replay rejection, and no bearer fallback. +- `workdir`'s `http-client` tests protect awaited fresh authorization and exact + request claims before any HTTP send. + +These fixtures use temporary authority data and observable synchronization, not +live databases or a fixed sleep to guess completion. This change does not claim a +latency benchmark or a measured production incident. diff --git a/docs/report/2026-10-12-t748-suite-diagnosis.md b/docs/report/2026-10-12-t748-suite-diagnosis.md new file mode 100644 index 000000000..3e73e67c2 --- /dev/null +++ b/docs/report/2026-10-12-t748-suite-diagnosis.md @@ -0,0 +1,166 @@ +# T-748: default-parallel suite failure diagnosis + +## Sources and preserved observations + +- Original local candidate: `61533fadb6b39465af60cb55aed2b4d0650319cd`. +- Comparison source: `56795fb92cb9def9bcd701bd51d92e99526f8eff` (develop/T742). +- Same implementation branch: `work/T-748-bounded-db-execution`. +- The original `target/t748-server-tests.log` remains untouched. Its 97 FAILED + lines and 600-second timeout were symptoms, not proof of one cause. +- Initial diagnosis used `cargo test -p yoi-workspace-server -- --nocapture`, + default libtest parallelism, without changing queue limits, features, or startup + settings. `target/t748-diagnostic-environment.txt` records the environment; + this host reports 24 available CPUs. Diagnostic captures were bounded to + 90/150 seconds to obtain errors before the original timeout, not counted as + successful full-suite runs. + +Logs under `target/` are local diagnostic artifacts, not tracked review inputs. +This report records their meaning and the permanent regressions that reproduce it. + +## 1. Different fixtures consumed one global Host admission budget + +The first non-intentional errors in `target/t748-server-nocapture.log` were +`DbCapacityExceeded` in manual Worker setup (`server.rs` at the then-current +line 50614), followed by assignment fixture construction (52990/53045) and +HTTP 503 responses. The deliberate before/after-commit executor panic tests +are different: those passed and verified unknown-outcome diagnostics. + +Temporary submit/start/cleanup instrumentation recorded submitting test names, +closure types, actual job IDs and the execution-owner identity. It showed a +single Host lane with `max=18`, with accepted jobs from unrelated fixtures. +A rejection snapshot contained `{32,45,46,...,61}`. The complete diagnostic +capture later contained cleanup for every submitted Host job on that owner; +it did not show permanently abandoned accepted Host jobs. Instrumentation was +removed before the source was committed/published. + +`target/t748-live-process-18.txt` sampled the test process and threads at the +actual rejection. That sample had 126 threads; the Host workers were runnable, +and the process was using approximately 17 CPU cores. There was no captured +thread-creation EAGAIN/unavailable error. This evidence does not establish an +OS thread/CPU hard limit as the cause. No test processes remained after the +bounded diagnostic run or the completed suite. + +A representative failure passed alone (0.56 seconds); the eight logging cases +passed with four test threads (2.46 seconds). The isolated comparison source's +full default-parallel suite passed: 931 library tests, one pre-existing ignored +migration fixture, 11 binary tests; library duration 168.74 seconds. This did +not prove that all new failures were capacity-related, but did distinguish the +new shared-admission rejection from an observed baseline failure. + +### Ownership fix (`f3d4aba5`) + +Production budgets remain shared and finite: DB 1 worker/32 queued, Host 2/16, +cleanup 2/16. Ordinary fixtures now explicitly own a finite DB executor and lazy +Host/cleanup owners at those same limits. API clones, child Workspace APIs, and +weak retention adapters use their injected owner. No global fixture lane cache, +parallelism reduction, unlimited queue, or capacity-wait retry was added. + +The permanent saturated-Host regression fills both running slots and all 16 +queued slots, cancels waiters, advances paused Tokio time, and still observes +rejection until actual completion. Another fixture and the separate cleanup lane +can progress; draining restores the original budget and completes exactly 18 +accepted jobs. Existing real-SQLite cancellation/lease/drain tests remain active. + +## 2. SQL offload invalidated fixture assumptions about first poll and publication + +After Host isolation the default-parallel suite finished in 171.57 seconds: +954 passed, three failed, one existing ignored. Actual errors were: + +- removal subscriber saw a legitimate queued Runtime stop Upsert before Removed; +- external session-open expiry exceeded its one-second bound while its DB lane + was still shared across other fixtures; +- grant replacement returned success because one initial poll had only submitted + the offloaded authority read, not reached the held original-grant lock. + +The ordinary DB fixtures were isolated too. The one-second expiry assertion was +not increased. Grant testing now drains its actual authority read then polls the +saved response to reach the held lock before replacing the grant. Removal testing +observes the committed stopped projection before taking the removal baseline. +The Removed payload assertion is unchanged. These are observable synchronization, +not sleeps or retries. Focused expiry, grant, and removal tests all passed. + +A captured cleanup job also attempted to poll a Tokio timer after its fixture +runtime had shut down. The terminal checker fixture now retains that original +runtime through accepted Host and cleanup jobs, then stops projection producers +and drains actual DB work before ending. Terminal SQL state is not treated as +actual cleanup completion. The subsequent full run contained only the three +intentional executor panic diagnostics, not the shutdown-context panic. + +The synchronization/teardown changes are in `df9d056d`. + +## 3. Restore intent's DEFERRED read-to-write upgrade returned SQLite BUSY + +The next default-parallel run finished in 171.48 seconds with 956 passed, two +failed, one existing ignored. Both failures were SQLite `DatabaseBusy`, extended +code 5, not capacity rejection: + +- production-profile zero-input Orchestrator restore; +- pending manual Worker claim restore reconciliation (500 instead of 409). + +Both failed alone too. `target/t748-busy-origin.log` captured the error and a +backtrace through `pin_internal_worker_restore_intent` -> the transaction helper +-> internal restore -> manual claim. The journal read authority first in a +DEFERRED transaction while the offloaded projection's other SQLite connection +could write. Such an upgrade can return BUSY without invoking SQLite's busy +handler; treating this as a retryable whole mutation would be incorrect. + +A new real-SQLite regression holds an IMMEDIATE writer on another connection, +registers an owned per-connection busy observer, and attempts one journal pin. +The old helper fails deterministically with BUSY/5 before waiting for ownership +(`target/t748-deferred-upgrade-red.log`, 0.23 seconds). The fixed helper waits +before its first authority read, while a current-thread timer and control task +progress. After releasing the writer it persists exactly one requested tuple; +there is no application-level mutation retry. + +`f597f1b5` makes this existing mutation helper explicitly +`with_immediate_transaction`. Restore journal and deletion authority reads, +mutation and commit remain in one transaction. Focused DB/HTTP tests (8), the +original restore and claim cases, and deletion tests (13) passed. + +## 4. Why a pre-dispatch failure could become the 600-second suite timeout + +The original long-running logging case joined an HTTP request with a fake provider +waiting for command dispatch. On pre-dispatch rejection the HTTP future can finish +with 503 while the receiver remains Pending: its channel is still owned by the +fixture, but no command will arrive. An unbounded `join!` waits forever in that +state and hides the actual error behind the outer suite timeout. + +The permanent `saturated_host_before_command_start_returns_availability_without_a_provider_operation` +regression uses the real signed HTTP route with a controlled full Host budget. It +observes the final 503/`db_capacity_exceeded` response and the still-live Pending +provider receiver, with no process or command dispatch. The original logging +fixture now asserts an early HTTP response before waiting for both branches, so +an unexpected rejection reports its body immediately rather than hanging. This +reproduces the causal timeout mechanism; the old log itself has no stack trace +from the hung provider future. + +The nine logging cases pass together at default parallelism (`49bfbfb2`). + +## Artifact/source isolation during comparison + +Initially the archived baseline used the existing Cargo target directory. That +replaced a same-package artifact/fingerprint; one later post-fix representative +command ran the baseline's 932-test binary (and `db_executor --list` had no tests). +That result was rejected as candidate evidence. All later candidate checks/tests +use `--target-dir target/t748-candidate-build`, separate from the baseline cache. +This changes artifact location, not test parallelism, features or admission policy. +The fresh candidate build listed the expected executor tests and total test count. + +## Validation and remaining composition boundary + +Final code source `49bfbfb2` passed the full default-parallel Server suite, +including the command-overload case: 960 passed, zero failures, one existing +ignored migration fixture; 11 binary tests, library duration 175.29 seconds. +The only captured panics were the three deliberate unknown-outcome tests. +Root `cargo check` compile closure passed (including TUI/Server), as did Workdir +default (153 tests) and `http-client` (156 tests) full suites, format/diff checks +and state-contract names (558 matches, zero unapproved, zero stale exceptions). +These commands used the isolated candidate artifact directory; libtest default +parallelism and the production budgets were unchanged. The MR records the exact +published source, which also includes this report and the boundary document. + +No latency benchmark, production incident measurement, target integration, +Ticket conclusion, or running-environment update is claimed. T743 has now +published its caller inventory/source; its owner will compose the T748 facade and +new callers, then obtain review of that exact combined source. T748 does not copy +those domain APIs or wait on the former unpublished branch. diff --git a/docs/report/T-743-T748-callers.md b/docs/report/T-743-T748-callers.md index 510ee2fd0..7e99259a8 100644 --- a/docs/report/T-743-T748-callers.md +++ b/docs/report/T-743-T748-callers.md @@ -2,6 +2,12 @@ ## Source identity and composition +This is the retained pre-composition caller inventory published with T-743 +`9b0d42fdbc2920736c3418bb894f26140e962bbc`. Its descriptions of direct callers +below identify the work to move, not the current execution placement. The later +normal merge, actual DB/Host placement, ten new regressions, full-suite evidence +and unresolved upstream review findings are in `T-743-T748-composition.md`. + This inventory belongs to the T-743 commit carrying this file on `work/T-743-ticket-public-surfaces` (T-742 base `56795fb92cb9def9bcd701bd51d92e99526f8eff`). Its exact published SHA and MR ID are recorded in the Ticket publication handoff. It is not a copy of the unpublished T-748 adapter and is not evidence that async DB execution has been integrated or reviewed. Normal-merge the validated public T-748 source into this existing work branch. Keep the MR's source/target selectors. Recheck the concrete new call graph below against the merged implementation; no placeholder executor, second connection authority, transaction split, fresh integration branch or force push is needed. diff --git a/docs/report/T-743-T748-composition.md b/docs/report/T-743-T748-composition.md new file mode 100644 index 000000000..e3b4d1757 --- /dev/null +++ b/docs/report/T-743-T748-composition.md @@ -0,0 +1,130 @@ +# T-743 / T-748 composition: execution-boundary evidence + +## Source and status + +T-743 continues on `work/T-743-ticket-public-surfaces`, MR +`01a12d19-b61b-79a2-baca-89241fd3c3d3` → `develop`. The published T-748 +source `66bb9d0b825d72e20923003b273b787007f3aed8` was normally fetched and +merged into that same source branch, without conflicts, replacement selectors, +force push, or a copied execution facade. The composition commit carrying this +report also connects the added callers below. Its exact published hash is in the +Ticket handoff and provider-resolved MR source. + +**This is a validated intermediate composition, not final independent approval.** +T-748 MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq3 requests changes on this +upstream source. Its two major findings are not fixed by the new assignment +caller work: `T748-DB-NETWORK-LANE` (Subjektiv mixed Runtime/DB placement) and +`T748-ASYNC-PROTOCOL-SIGNING` (cold protocol observation source signing). +That authoritative verdict was checked again after the combined full suite; +no newer published upstream source or verdict was present. The T-748 owner was +informed through its Ticket of the composition and requested to provide the +normally advanced validated fix source. Do not interpret a green suite as proof +that those untested boundaries are safe. Normal-merge that fix into this same +T-743 source, revalidate, and independently review the exact combined source. + +## Caller placement + +- Canonical and Browser assignment pages and sanitized diagnostics share one + implementation. Workspace scope, Ticket reference resolution and each whole + synchronous Store read run on the shared bounded DB owner. None is left as a + synchronous pre-call on a Tokio request thread. +- Typed Ticket query, including principal/handling/current-closure filtering + before pagination, retains T-748's whole authority/read/enrichment DB job. + The regression holds the **authority's actual candidate connection**, not the + separate API Store connection used by only some enrichment paths. This also + proves the empty-result path and wrong-principal filters offload correctly. +- Account assignment answers derive attribution from Backend-authenticated + context inside the owned DB closure. Runtime answers preserve the live current + Orchestrator discovery check on the finite Host lane, because Runtime worker + lookup can perform remote HTTP. The DB closure derives Runtime attribution + without I/O and the unchanged answer helper validates current singleton and + live binding **again under its IMMEDIATE write transaction**, including replay. + Target principal and request-body prose never supply author authority. +- The same answer helper keeps source authority, actionable-assignment gate, + assignment/event CAS, full operation fingerprint/replay, body, audit, receipt + and commit together. There is no await within that transaction and no automatic + mutation retry. +- The accepted SQL job admits the existing finite Host attention continuation + **after commit and before returning its result**. It inherits the request + lifecycle source, owns lookup/Runtime dispatch independently of the waiter and + does not wait on Runtime/network on the sole DB worker. A live HTTP waiter + observes best-effort notification completion; losing the waiter does not + cancel accepted SQL or accepted publication. Rejected Host publication retains + the existing saved-input scan/recovery and diagnostic behavior, not rollback + or a synthetic successful delivery. +- Normal Tools/WIP still converge on the canonical HTTP contracts. No second + ledger, query adapter, source authority, read pool or delivery implementation + was added. Existing T-748 set/clear/start/delivery boundaries remain intact. + +## New regression proof + +`crates/workspace-server/src/server_assignment_db_execution_tests.rs` contains +10 tests on fixture-owned bounded DB/Host owners, temporary databases and explicit +observable gates. Parent inspected the child's committed tool output, the actual +new test code and then the combined default-parallel suite, rather than treating +an end notification as proof. The child's delegated file scope was released. + +The regressions prove: + +1. Assignment/diagnostic read and Ticket reference resolution wait off the + current-thread executor on their actual Store connection mutex. +2. Typed principal/handling/closure query matrices, including empty results, + hold the candidate connection and filter before the page limit. +3. A real second SQLite connection's IMMEDIATE writer reaches the answer's owned + busy observer while the same Tokio runtime's timer and independent control + progress; one transaction preserves the answer/event/audit/receipt. +4. Saturated DB admission returns HTTP 503 and does not accept an answer. +5. Cancelling an accepted answer waiter retains its actual job slot/lifecycle + through SQL and Host publication; body/audit/receipt persist, dispatch is once, + and actual cleanup releases all observed leases. +6. Successful HTTP answer waits for its admitted publication without occupying + the DB worker; an independent DB callback can progress during publication. +7. Account and Browser receipt replay, stale CAS, operation mismatch and bad + cursor/error mapping retain their domain meanings. +8. Real signed non-Orchestrator source and signed proof replay are denied with + no answer/receipt; signed current Orchestrator canonical/Browser replay shares + one source-bound receipt, and another source cannot replay it. +9. Saturated **2-worker/16-queued Host** live-discovery rejects canonical/Browser + Runtime answers before SQL admission, without receipt/audit/publication or + retained lifecycle, while independent DB callbacks progress. This last test + injects authenticated context at generated handlers to isolate discovery + admission; the preceding tests exercise actual signed-proof middleware. + +Read/query contention intentionally uses the actual connection mutex; answer +contention uses a separate real SQLite writer. Neither is described as a full +remote Runtime latency benchmark. No fixed completion sleeps, mutation retry, +parallelism reduction, process-global fixture budgets or new ignored tests were +introduced. Fixtures drain accepted Host/cleanup jobs, shut down projection +producers, then drain DB jobs before ending their original Tokio driver. Owned +busy-handler userdata and SQLite locks unregister/release through RAII. + +## Validation on this composition + +- Root `cargo check`: passed, normal default-member compile closure. +- `cargo test -p yoi-workspace-server --lib assignment_db_execution_tests`: + **10 passed, 0 failed**, 2.63s (child committed output). Parent reran the first + nine after separating Runtime discovery and included all ten in the full suite. +- `cargo test -p yoi-workspace-server`: original default parallelism, + **985 library passed, 0 failed, 1 pre-existing ignored**, 176.60s; + **11 binary passed**, doc tests passed. Log: + `target/t743-combined-server-suite.log`. +- `cargo test -p workdir`: **153 passed** across library/integration targets; + `cargo test -p workdir --features http-client`: **156 passed**, doc tests passed. + Logs: `target/t743-combined-workdir.log`, `target/t743-combined-workdir-http.log`. +- Combined Web check: **0 errors, 0 warnings**; static production build passed + with the existing chunk-size advisory. Web tests **613 passed**, components + **318 passed**, Chromium Ticket/principal browser tests **7 passed**. + Logs: `target/t743-combined-{web-check,web-build,web-tests,components,browser}.log`. + The last change is Server execution placement/test code, not a UI change. +- All six server-api TypeScript generator `--check` commands passed: ticket, + repository OpenAPI, runtime, workdir, Worker launch and companion. +- `cargo fmt --all -- --check`, `git diff --check HEAD` and + `env -u LD_LIBRARY_PATH deno run --allow-read --allow-run=git tools/check-state-contract-names.ts` + passed; naming: **558 matching lines, 0 unapproved, 0 stale exceptions**. + +Earlier T-743 public-surface, five-crate, Prompt/Tool, visual and real-Worker +fixture evidence is retained in `T-743-public-surface-validation.md`. Scripted +model/loopback browser fixtures do not attest real model judgment or all T-744 +live paths. Target integration, Ticket/MR conclusion, live update and retained +Worker/Workdir cleanup were not performed. Fresh independent combined-source +review remains pending the upstream fixes, not silently waived. diff --git a/docs/report/T-743-public-surface-validation.md b/docs/report/T-743-public-surface-validation.md index 43e04364e..fe3a06b37 100644 --- a/docs/report/T-743-public-surface-validation.md +++ b/docs/report/T-743-public-surface-validation.md @@ -26,7 +26,7 @@ Canonical `/workspaces/{workspace_id}/tickets/{ticket_id}` and `/api/w/{workspac ## Validation -Passed on the combined working snapshot: +Passed on the initial T-743 public-surface snapshot (before T-748 composition); later combined execution-boundary evidence is in `T-743-T748-composition.md`: - Root `cargo check` (normal TUI/Server compile closure). - `cargo test --no-fail-fast -p ticket -p worker-runtime -p worker -p yoi-workspace-server -p server-api`: all selected crate targets passed. Library counts: server-api 98; ticket 99; worker 994; worker-runtime 417; workspace-server 946 with one existing ignored test. Included integration/binary targets also passed. Log: `target/t743-changed-crates-final.log`. @@ -51,7 +51,7 @@ List/Orchestrator final evidence: `t743-list-final-reviewed-light` and `t743-lis ## Explicit remaining work and limits -T-748's DB execution facade is **not** copied or integrated. Its latest notified candidate is local-only and its full Server suite has unresolved failures. When T-748 publishes a validated source, normal-merge it, place each new whole synchronous domain helper/authority query into the common bounded DB execution boundary, keep the answer transaction intact, and leave Runtime/network waiting off the DB worker. Recheck cancellation/overload and commit-before-publication at the combined boundary. The exact combined published source then needs independent trusted review. +The published T-748 source `66bb9d0b825d72e20923003b273b787007f3aed8` has now been normally merged into the same T-743 source and the added callers connected to its existing bounded owner. The answer transaction remains intact; Runtime live discovery and post-commit dispatch use the finite Host lane. Combined full Server/Workdir and ten new boundary regressions pass; see `T-743-T748-composition.md` for exact proof and distinctions. However T-748 MR seq3 has two unresolved major findings on other Subjektiv/network and cold protocol signing paths. A validated upstream fix source, normal composition/revalidation and fresh independent exact combined-source review remain necessary. This is not final approval. Current evidence does not attest actual model judgment quality. Prompt/Tool fixtures are scripted inclusion/schema/transport tests; Chromium uses a loopback API fixture. The real-Worker Server fixture uses a scripted model with actual Controller/session/Runtime acceptance and canonical processing synchronization. None is a live migration/rollout, proof of actual processing for every production delivery, or completion of T-744's full real-path validation.