diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index 940506ca7..ae155b008 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -432,16 +432,23 @@ mod client { use super::*; use crate::{Workdir, WorkdirSession}; - /// Provides a fresh bearer token for each Runtime request. Backend - /// implementations can mint short-lived capability tokens without making a - /// Worker-bound session expire with the token used to open it. + /// An owned, Send authorization future that can offload synchronous authority + /// work without borrowing request data or the authorization provider. + pub type WorkdirHttpAuthorizationFuture = std::pin::Pin< + Box> + Send + 'static>, + >; + + /// Provides a fresh bearer token asynchronously for each Runtime request. + /// Backend implementations can mint short-lived capability tokens without + /// making a Worker-bound session expire with the token used to open it. + /// Blocking DB/signing work must be offloaded by the implementation. pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync { fn bearer_token( &self, - method: &str, - path_and_query: &str, - body: &[u8], - ) -> Result; + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture; } struct FixedBearerToken(Arc); @@ -455,11 +462,12 @@ mod client { impl WorkdirHttpAuthorization for FixedBearerToken { fn bearer_token( &self, - _method: &str, - _path_and_query: &str, - _body: &[u8], - ) -> Result { - Ok(self.0.to_string()) + _method: String, + _path_and_query: String, + _body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let token = self.0.clone(); + Box::pin(async move { Ok(token.to_string()) }) } } @@ -509,7 +517,9 @@ mod client { )?; let body = serde_json::to_vec(&request) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = authorization.bearer_token("POST", url.path(), &body)?; + let token = authorization + .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) + .await?; let response = client .post(url) .bearer_auth(token) @@ -560,7 +570,10 @@ mod client { let operation = WorkdirSessionOperationRequest { operation }; let body = serde_json::to_vec(&operation) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = self.authorization.bearer_token("POST", url.path(), &body)?; + let token = self + .authorization + .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) + .await?; let response = self .client .post(url) @@ -864,7 +877,10 @@ mod client { &self.base_url, &["v1", "workdir-sessions", self.session_id.as_str()], )?; - let token = self.authorization.bearer_token("DELETE", url.path(), &[])?; + let token = self + .authorization + .bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new()) + .await?; let response = self .client .delete(url) @@ -918,11 +934,124 @@ mod client { WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string()) } + #[cfg(test)] + mod authorization_tests { + use super::*; + use std::sync::Mutex; + + #[derive(Debug, Default)] + struct RejectAuthorization { + requests: Arc)>>>, + } + + impl WorkdirHttpAuthorization for RejectAuthorization { + fn bearer_token( + &self, + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let requests = self.requests.clone(); + Box::pin(async move { + tokio::task::yield_now().await; + let mut requests = requests.lock().unwrap(); + requests.push((method, path_and_query, body)); + Err(WorkdirError::Unavailable(format!( + "authorization rejected request {}", + requests.len() + ))) + }) + } + } + + #[tokio::test] + async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let base_url: Url = format!("http://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(); + let client = Client::builder().no_proxy().build().unwrap(); + let authorization = Arc::new(RejectAuthorization::default()); + let workdir = Workdir::new("w"); + let open_request = OpenWorkdirSessionRequest { + owner_worker_id: Some("worker".to_string()), + }; + let error = RemoteWorkdirSession::open_with_authorization( + client.clone(), + base_url.clone(), + authorization.clone(), + workdir.id().clone(), + open_request.clone(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 1" + )); + + // Seed an already-open session to exercise the public operation and + // close boundaries independently of the rejected open request. + let remote = RemoteWorkdirSession { + client, + base_url, + authorization: authorization.clone(), + workdir, + session_id: WorkdirSessionId::new("s").unwrap(), + capabilities: WorkdirSessionCapabilities::EMPTY, + closed: AtomicBool::new(false), + }; + let stat = StatRequest { + path: crate::WorkdirPath::new("file").unwrap(), + }; + let error = remote.stat(stat.clone()).await.unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 2" + )); + let error = remote.close().await.unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 3" + )); + assert_eq!( + *authorization.requests.lock().unwrap(), + vec![ + ( + "POST".to_string(), + "/v1/working-directories/w/sessions".to_string(), + serde_json::to_vec(&open_request).unwrap(), + ), + ( + "POST".to_string(), + "/v1/workdir-sessions/s/operations".to_string(), + serde_json::to_vec(&WorkdirSessionOperationRequest { + operation: WorkdirSessionOperation::Stat(stat), + }) + .unwrap(), + ), + ( + "DELETE".to_string(), + "/v1/workdir-sessions/s".to_string(), + Vec::new(), + ), + ] + ); + assert_eq!( + listener.accept().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); + } + } + pub use self::RemoteWorkdirSession as ClientSession; } #[cfg(feature = "http-client")] -pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization}; +pub use client::{ + ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture, +}; #[cfg(test)] mod tests { diff --git a/crates/workspace-server/src/assignment_delivery_tests.rs b/crates/workspace-server/src/assignment_delivery_tests.rs index b8702edb7..c8e2cb7f6 100644 --- a/crates/workspace-server/src/assignment_delivery_tests.rs +++ b/crates/workspace-server/src/assignment_delivery_tests.rs @@ -147,7 +147,7 @@ async fn assignment_delivery_response_loss_returns_immutable_payload_after_progr .update_ticket_assignment_tracking("w", "tracking", &update, &source) .unwrap(); drop(store); - let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); assert_eq!( store .list_pending_ticket_assignment_deliveries("w", 100) @@ -208,7 +208,7 @@ async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt() let path = directory.path().join("workspace.db"); let input = pending.input_event_id.clone(); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open(path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(path).unwrap(); barrier.wait(); store .reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source()) @@ -568,7 +568,7 @@ async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_co "original rendered prompt" ); drop(store); - let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); assert_eq!( store .prepare_ticket_assignment_delivery_content( @@ -634,7 +634,7 @@ fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_l let operations = stored_fixture_rows(&conn, "ticket_assignment_operations"); let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history"); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let pending = store .list_pending_ticket_assignment_deliveries("w", 100) .unwrap(); @@ -684,7 +684,7 @@ async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once let barrier = barrier.clone(); let path = directory.path().join("workspace.db"); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open(path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(path).unwrap(); barrier.wait(); delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source()) .unwrap() @@ -716,8 +716,8 @@ fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() { let conn = tracking_schema91_fixture(&path); migrate_ticket_tracking_v91_to_v92(&conn).unwrap(); drop(conn); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let objects = |store: &SqliteWorkspaceStore| { store.with_conn(|conn| { let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?; @@ -741,7 +741,7 @@ async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() { .unwrap(); drop(store); assert!( - matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) + matches!(SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) ); } diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 513daef26..5255267c7 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -78,7 +78,7 @@ async fn tracking_store_every_replay_fences_current_singleton_and_authenticated_ ); // Model the authority change after authenticated source capture, before // BEGIN IMMEDIATE on a second connection. Receipt presence cannot bypass it. - let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); second.with_conn_mut(|conn| { let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; match change { @@ -233,7 +233,7 @@ async fn tracking_store_same_principal_reissue_and_exact_retries_keep_original_r .is_err() ); drop(store); - let reopened = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); assert_eq!( reopened .get_ticket_assignment_tracking("w", "tracking", "a") @@ -441,7 +441,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .collect::>(), vec![92, 93] ); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, 93); @@ -525,7 +525,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .list_ticket_assignment_tracking_history("w", "pending", "a-pending") .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( reopened .list_ticket_assignment_tracking_history("w", "pending", "a-pending") @@ -550,7 +550,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first let releases_before = stored_fixture_rows(&conn, "ticket_assignment_work_releases"); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "w").unwrap(); let reopen = || { let observed = backend.show("historical-done".into()).unwrap(); @@ -645,7 +645,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first ); // A second observer holds the same Assign baseline. Only the first // explicit writer may consume it; BEGIN IMMEDIATE serializes the CAS. - let second = SqliteWorkspaceStore::open(&path).unwrap(); + let second = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( second .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") @@ -740,7 +740,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first }).unwrap(); drop(second); drop(store); - let reopened_store = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened_store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( reopened_store .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") @@ -796,7 +796,7 @@ fn tracking_store_schema91_missing_assign_audit_remains_unknown_without_cas_base ) .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let unknown = store .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") .unwrap() @@ -1068,7 +1068,7 @@ async fn orchestrator_assignment_all_runtime_retries_fence_singleton_and_binding &source, ) .unwrap(); - let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); second.with_conn_mut(|conn| {let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;match change { "replace" => {tx.execute("UPDATE worker_singleton_owners SET worker_id='replacement',generation=2 WHERE workspace_id='w'",[])?;}, "revoke" => {tx.execute("UPDATE workspace_runtime_bindings SET revoked_at='revoked' WHERE workspace_id='w'",[])?;}, diff --git a/crates/workspace-server/src/auth.rs b/crates/workspace-server/src/auth.rs index aa202f2de..b5badef0d 100644 --- a/crates/workspace-server/src/auth.rs +++ b/crates/workspace-server/src/auth.rs @@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option { }) } -pub async fn resolve_request_actor( - store: &S, +pub async fn resolve_request_actor( + store: std::sync::Arc, headers: &HeaderMap, cookie_name: &str, ) -> Result> { - if let Some(token) = parse_bearer(headers) { + let bearer = parse_bearer(headers); + let session = parse_cookie(headers, cookie_name); + if bearer.is_none() && session.is_none() { + return Ok(None); + } + let executor = store.db_executor(); + executor + .run(move || resolve_request_actor_sync(store.as_ref(), bearer, session)) + .await +} + +// One owned synchronous stage includes token usage marking and the user read. +// Do not call async store wrappers here: they use the same executor themselves. +fn resolve_request_actor_sync( + store: &dyn ControlPlaneStore, + bearer: Option, + session: Option, +) -> Result> { + if let Some(token) = bearer { let token_hash = token_hash(&token); if let Some(api_token) = store.resolve_api_token(&token_hash)? { if api_token @@ -109,7 +127,7 @@ pub async fn resolve_request_actor( } } - if let Some(session_token) = parse_cookie(headers, cookie_name) { + if let Some(session_token) = session { if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? { if is_expired(&session.expires_at) { return Ok(None); @@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error { #[cfg(test)] mod tests { use super::*; + use crate::store::{ + AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord, + }; + use std::sync::Arc; + + fn seeded_store() -> Arc { + let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::in_memory_for_test() + .unwrap() + .with_db_executor(executor), + ); + store + .upsert_account(&AccountRecord { + account_id: "account-auth".into(), + kind: "user".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .upsert_user(&UserRecord { + user_id: "user-auth".into(), + account_id: "account-auth".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .create_browser_session(&BrowserSessionRecord { + session_id: "session-auth".into(), + token_hash: token_hash("session-secret"), + user_id: "user-auth".into(), + created_at: "1".into(), + expires_at: "9999-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + store + } + + #[tokio::test] + async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() { + let store = seeded_store(); + let hash = token_hash("api-secret"); + store + .create_api_token(&ApiTokenRecord { + token_id: "token-auth".into(), + token_hash: hash.clone(), + user_id: "user-auth".into(), + label: "Auth".into(), + created_at: "1".into(), + expires_at: None, + revoked_at: None, + last_used_at: None, + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap()); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let actor = resolve_request_actor(store.clone(), &headers, "session") + .await + .unwrap() + .unwrap(); + assert_eq!(actor.user_id, "user-auth"); + assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken); + assert!( + store + .resolve_api_token(&hash) + .unwrap() + .unwrap() + .last_used_at + .is_some() + ); + } + + #[tokio::test] + async fn expired_api_token_does_not_fall_back_to_valid_browser_session() { + let store = seeded_store(); + let hash = token_hash("expired-secret"); + store + .create_api_token(&ApiTokenRecord { + token_id: "token-auth".into(), + token_hash: hash.clone(), + user_id: "user-auth".into(), + label: "Auth".into(), + created_at: "1".into(), + expires_at: Some("2000-01-01T00:00:00Z".into()), + revoked_at: None, + last_used_at: None, + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + "Bearer expired-secret".parse().unwrap(), + ); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + assert!( + resolve_request_actor(store.clone(), &headers, "session") + .await + .unwrap() + .is_none() + ); + assert!( + store + .resolve_api_token(&hash) + .unwrap() + .unwrap() + .last_used_at + .is_none() + ); + } + + #[tokio::test] + async fn unknown_bearer_can_fall_back_to_valid_browser_session() { + let store = seeded_store(); + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + "Bearer unknown-secret".parse().unwrap(), + ); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let actor = resolve_request_actor(store, &headers, "session") + .await + .unwrap() + .unwrap(); + assert_eq!(actor.user_id, "user-auth"); + assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession); + } + + #[tokio::test] + async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() { + let store = seeded_store(); + let executor = store.db_executor(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let blocker_executor = executor.clone(); + let blocker = tokio::spawn(async move { + blocker_executor + .run(move || -> Result<()> { + let _ = started.send(()); + release_rx.recv().unwrap(); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(1), started_rx) + .await + .unwrap() + .unwrap(); + let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) })); + assert!(futures::poll!(queued.as_mut()).is_pending()); + let mut headers = HeaderMap::new(); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + resolve_request_actor(store, &headers, "session"), + ) + .await; + release.send(()).unwrap(); + blocker.await.unwrap().unwrap(); + queued.await.unwrap(); + assert!(matches!( + result.expect("capacity rejection must not wait for queued SQL"), + Err(Error::DbCapacityExceeded) + )); + } #[test] fn user_code_uses_the_uuid_random_tail() { diff --git a/crates/workspace-server/src/authority.rs b/crates/workspace-server/src/authority.rs index 1203f2503..e590d2177 100644 --- a/crates/workspace-server/src/authority.rs +++ b/crates/workspace-server/src/authority.rs @@ -3349,7 +3349,7 @@ mod tests { async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -3424,7 +3424,7 @@ mod tests { async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -3557,7 +3557,7 @@ mod tests { write_ticket(dir.path(), "00000000001J5", "Second ticket", "queued"); write_ticket(dir.path(), "00000000001J6", "Third ticket", "planning"); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), @@ -3989,7 +3989,7 @@ VALUES ('workspace-test', 'ticket', 4); async fn objective_mutations_write_sqlite_records_and_audit_events() { let dir = tempfile::tempdir().unwrap(); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), diff --git a/crates/workspace-server/src/config_source.rs b/crates/workspace-server/src/config_source.rs index 58158f464..7ac608622 100644 --- a/crates/workspace-server/src/config_source.rs +++ b/crates/workspace-server/src/config_source.rs @@ -795,7 +795,7 @@ mod tests { } async fn open_store() -> SqliteWorkspaceStore { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); store } @@ -1454,7 +1454,7 @@ mod tests { #[tokio::test] async fn invalid_candidate_is_never_persisted() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let main = current.snapshot.get(&path(MAIN_CONFIG_ENTRYPOINT)).unwrap(); @@ -1478,7 +1478,7 @@ mod tests { #[tokio::test] async fn valid_candidate_commits_snapshot_content_and_provenance_atomically() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let committed = store @@ -1496,7 +1496,7 @@ mod tests { #[tokio::test] async fn stale_cas_cannot_overwrite_newer_tree() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let request = commit_request(¤t, vec![update_main(¤t, "{ answer = 42; }")]); @@ -1514,7 +1514,7 @@ mod tests { #[tokio::test] async fn committed_content_remains_retrievable_after_later_commit() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let first = store @@ -1772,7 +1772,7 @@ mod tests { #[test] fn migration_creates_config_authority_tables() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .with_conn(|conn| { for table in [ diff --git a/crates/workspace-server/src/db_executor.rs b/crates/workspace-server/src/db_executor.rs new file mode 100644 index 000000000..1c44ef8c9 --- /dev/null +++ b/crates/workspace-server/src/db_executor.rs @@ -0,0 +1,957 @@ +//! Bounded execution of whole synchronous DB operations, independent of Tokio. +//! +//! Accepted jobs belong to the workers, not their async waiters. Cancelling a +//! waiter does not cancel or retry SQL, release capacity, or release its lease. +use std::{ + collections::BTreeSet, + future::Future, + num::NonZeroUsize, + panic::{AssertUnwindSafe, catch_unwind}, + sync::{Arc, Mutex, OnceLock, mpsc}, +}; +use tokio::sync::{Notify, oneshot}; + +use crate::{Error, Result}; + +/// The default is one FIFO worker and at most 32 waiting operations. +#[derive(Clone, Copy, Debug)] +pub struct DbExecutorLimits { + pub workers: NonZeroUsize, + pub queued: NonZeroUsize, +} + +impl Default for DbExecutorLimits { + fn default() -> Self { + Self { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(32).unwrap(), + } + } +} + +/// A request can retain its lifecycle authority until actual SQL finishes. +/// Retention happens at submission, while the caller's task-local scope exists. +pub(crate) trait DbOperationLifetime: Send + Sync { + fn retain(&self) -> Box; +} + +tokio::task_local! { + static OPERATION_LIFETIME: Arc; +} + +pub(crate) async fn with_operation_lifetime( + lifetime: Arc, + future: F, +) -> F::Output { + OPERATION_LIFETIME.scope(lifetime, future).await +} + +#[derive(Default)] +struct Accepted { + last_job_id: u64, + unfinished: BTreeSet, +} + +#[derive(Default)] +struct Tracking { + accepted: Mutex, + finished: Notify, +} + +struct Completion { + tracking: Arc, + id: u64, +} + +impl Drop for Completion { + fn drop(&mut self) { + self.tracking + .accepted + .lock() + .unwrap() + .unfinished + .remove(&self.id); + self.tracking.finished.notify_waiters(); + } +} + +struct Job { + operation: Box, + // Field drop order releases authority before completion becomes observable. + _lifetime: Option>, + lifetime_source: Option>, + _completion: Completion, + runtime: Option, + diagnostics: tracing::Dispatch, + span: tracing::Span, +} + +fn report_unknown(accepted_job_id: u64, phase: &'static str) { + tracing::error!( + code = "db_operation_outcome_unknown", + accepted_job_id, + phase, + error = %Error::DbOperationOutcomeUnknown, + "DB job panicked; outcome unknown, operation will not be retried" + ); +} + +/// Clones share the queue and actual-job budget. Workers retain only the +/// receiver, so dropping the last sender closes the queue; accepted jobs finish +/// before the dedicated threads exit. No thread is created per store open. +#[derive(Clone)] +pub struct DbExecutor { + sender: Option>, + tracking: Arc, + max_jobs: usize, +} + +impl DbExecutor { + pub fn new(limits: DbExecutorLimits) -> Result { + let max_jobs = limits + .workers + .get() + .checked_add(limits.queued.get()) + .ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?; + let (sender, receiver) = mpsc::sync_channel::(limits.queued.get()); + let receiver = Arc::new(Mutex::new(receiver)); + for index in 0..limits.workers.get() { + let receiver = receiver.clone(); + std::thread::Builder::new() + .name(format!("workspace-db-{index}")) + .spawn(move || { + loop { + let next = receiver.lock().unwrap().recv(); + let Ok(job) = next else { break }; + let Job { + operation, + _lifetime: lifetime, + lifetime_source, + _completion: completion, + runtime, + diagnostics, + span, + } = job; + // Keep diagnostics with their submitting subscriber as + // well as the runtime; no process-global override. + let _diagnostics = tracing::dispatcher::set_default(&diagnostics); + let _span = span.enter(); + // Preserve the submitting runtime for synchronous Host + // post-commit hooks, including reply/lease destructors. + // enter() supplies context only: never drive a future + // or await/nest an executor submission in a transaction. + let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter); + // A cancelled waiter's discarded result may itself have + // a destructor. Keep both the lease and actual-job slot + // until that cleanup finishes, even if it panics. + let execute = || { + if catch_unwind(AssertUnwindSafe(operation)).is_err() { + report_unknown(completion.id, "reply_cleanup"); + } + if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() { + report_unknown(completion.id, "lifetime_cleanup"); + } + }; + // Synchronous post-commit continuations can admit another + // bounded job. Transfer the original lifecycle source as + // well, so its authority survives the parent job/waiter. + // This does not inject context into detached Tokio tasks. + if let Some(source) = lifetime_source { + OPERATION_LIFETIME.sync_scope(source, execute); + } else { + execute(); + } + drop(completion); + } + }) + .map_err(|_| Error::DbExecutorUnavailable)?; + } + Ok(Self { + sender: Some(sender), + tracking: Arc::default(), + max_jobs, + }) + } + + pub fn shared() -> Self { + static SHARED: OnceLock = OnceLock::new(); + SHARED + .get_or_init(|| { + Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self { + sender: None, + tracking: Arc::default(), + max_jobs: 0, + }) + }) + .clone() + } + + /// Submit without waiting for queue capacity. The closure must contain the + /// entire synchronous operation/transaction, with no nested async runtime. + /// The submitting Tokio handle, when present, is entered for synchronous + /// post-commit hooks and worker cleanup; it is never used to block_on SQL. + pub async fn run( + &self, + operation: impl FnOnce() -> std::result::Result + Send + 'static, + ) -> std::result::Result + where + T: Send + 'static, + E: From + Send + 'static, + { + self.submit(operation)? + .await + .unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into())) + } + + /// Admit a fire-and-forget operation immediately, without spawning a Tokio + /// task or waiting for capacity. Err means the operation was not accepted; + /// Ok means the worker owns it through execution and cleanup, even after + /// the submitting task is cancelled. A panic emits a structured unknown- + /// outcome diagnostic, never an automatic retry. drain() tracks these jobs. + pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> { + self.submit(move || { + operation(); + Ok::<_, Error>(()) + }) + .map(drop) + } + + /// Shared synchronous admission: neither dropped receivers nor cancelled + /// callers remove accepted work from the queue or the actual-job budget. + fn submit( + &self, + operation: impl FnOnce() -> std::result::Result + Send + 'static, + ) -> std::result::Result>, E> + where + T: Send + 'static, + E: From + Send + 'static, + { + let sender = self + .sender + .as_ref() + .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; + let runtime = tokio::runtime::Handle::try_current().ok(); + let diagnostics = tracing::dispatcher::get_default(Clone::clone); + let span = tracing::Span::current(); + let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok(); + let lifetime = lifetime_source.as_ref().map(|source| source.retain()); + let (reply, result) = oneshot::channel(); + // Serialize admission and acceptance snapshots. Worker completion cannot + // race acceptance, and cancelled waiters never alter this actual-job cap. + { + let mut accepted = self.tracking.accepted.lock().unwrap(); + if accepted.unfinished.len() >= self.max_jobs { + return Err(Error::DbCapacityExceeded.into()); + } + let id = accepted + .last_job_id + .checked_add(1) + .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; + // Never reuse the identity of a rejected job: its destructor may run + // after another submission has acquired the admission lock. + accepted.last_job_id = id; + let job = Job { + operation: Box::new(move || { + let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| { + report_unknown(id, "operation"); + Err(Error::DbOperationOutcomeUnknown.into()) + }); + let _ = reply.send(outcome); + }), + _lifetime: lifetime, + lifetime_source, + runtime, + diagnostics, + span, + _completion: Completion { + tracking: self.tracking.clone(), + id, + }, + }; + match sender.try_send(job) { + Ok(()) => { + accepted.unfinished.insert(id); + } + Err(error) => { + // The rejected Job's completion also locks tracking; drop it only + // after releasing the admission lock. + drop(accepted); + return Err(match error { + mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded, + mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable, + } + .into()); + } + } + } + Ok(result) + } + + /// Wait for every job accepted before this barrier is first polled (not + /// later submissions). This bypasses queue capacity and survives cancelled + /// job waiters. It does not close the queue or stop new submissions. + pub async fn drain(&self) -> Result<()> { + if self.sender.is_none() { + return Err(Error::DbExecutorUnavailable); + } + let through = self.tracking.accepted.lock().unwrap().last_job_id; + loop { + let notified = self.tracking.finished.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self + .tracking + .accepted + .lock() + .unwrap() + .unfinished + .first() + .is_none_or(|id| *id > through) + { + return Ok(()); + } + notified.await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::{Connection, TransactionBehavior, ffi}; + use std::{ffi::c_void, time::Duration}; + + fn executor(queued: usize) -> DbExecutor { + DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(queued).unwrap(), + }) + .unwrap() + } + + fn database() -> (tempfile::TempDir, Connection, Connection) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite"); + let owner = Connection::open(&path).unwrap(); + owner + .execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)") + .unwrap(); + let worker = Connection::open(&path).unwrap(); + (dir, owner, worker) + } + + struct BusyGate { + waiting: Option>, + release: mpsc::Receiver<()>, + } + + // A fixture-owned callback proves SQLite actually reached its lock wait. + // Unlike busy_handler's fn pointer API, the SQLite API carries a per-call + // context, so tests need neither process-global state nor guessed sleeps. + struct BusyRegistration { + db: *mut ffi::sqlite3, + _gate: Box, + } + + impl BusyRegistration { + fn install(conn: &Connection, gate: BusyGate) -> Self { + unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 { + // The Box remains registered only while this connection and + // BusyRegistration are alive, on this same dedicated worker. + let gate = unsafe { &mut *context.cast::() }; + if let Some(waiting) = gate.waiting.take() { + let _ = waiting.send(()); + if gate.release.recv().is_ok() { + return 1; + } + } + 0 + } + let mut gate = Box::new(gate); + // SAFETY: conn owns the handle; registration is dropped before conn. + let db = unsafe { conn.handle() }; + let result = unsafe { + ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast()) + }; + assert_eq!(result, ffi::SQLITE_OK); + Self { db, _gate: gate } + } + } + + impl Drop for BusyRegistration { + fn drop(&mut self) { + // SAFETY: unregister before releasing the callback's boxed context. + unsafe { + ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut()); + } + } + } + + async fn observed(future: impl Future) -> T { + tokio::time::timeout(Duration::from_secs(5), future) + .await + .expect("DB job did not reach the expected observable state") + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() { + let (_dir, mut owner, mut worker) = database(); + let lock = owner + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let executor = executor(1); + let (waiting, blocked) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut write = Box::pin(executor.run(move || -> Result<()> { + let _busy = BusyRegistration::install( + &worker, + BusyGate { + waiting: Some(waiting), + release: permit, + }, + ); + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + Ok(()) + })); + assert!(futures::poll!(&mut write).is_pending()); + observed(blocked).await.unwrap(); + let control = tokio::spawn(async { 42 }); + assert_eq!(observed(control).await.unwrap(), 42); + // The timer itself is the contract under test, not synchronization. + observed(tokio::time::sleep(Duration::from_millis(1))).await; + assert!(futures::poll!(&mut write).is_pending()); + lock.rollback().unwrap(); + release.send(()).unwrap(); + observed(write).await.unwrap(); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + } + + struct Lease(Arc); + struct Retained(Arc); + impl DbOperationLifetime for Lease { + fn retain(&self) -> Box { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Box::new(Retained(self.0.clone())) + } + } + impl Drop for Retained { + fn drop(&mut self) { + self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() { + let (_dir, mut owner, mut worker) = database(); + let lock = owner + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let executor = executor(1); + let (waiting, blocked) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut running = Box::pin(executor.run(move || -> Result<()> { + let _busy = BusyRegistration::install( + &worker, + BusyGate { + waiting: Some(waiting), + release: permit, + }, + ); + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + Ok(()) + })); + assert!(futures::poll!(&mut running).is_pending()); + observed(blocked).await.unwrap(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap(); + let mut queued = Box::pin(with_operation_lifetime( + lease, + executor.run(move || -> Result<()> { + let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + // FIFO is observable in durable SQL, not inferred from waiter order. + let first: i64 = + tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| { + row.get(0) + })?; + assert_eq!(first, 1); + tx.execute("INSERT INTO writes VALUES (2)", [])?; + tx.commit()?; + Ok(()) + }), + )); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!(matches!( + executor.run(|| Ok::<_, Error>(())).await, + Err(Error::DbCapacityExceeded) + )); + let mut drained = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + lock.rollback().unwrap(); + release.send(()).unwrap(); + observed(running).await.unwrap(); + observed(drained).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() { + let (dir, owner, mut parent_conn) = database(); + let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap(); + let parent = executor(1); + let child = executor(1); + let child_owner = child.clone(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let source: Arc = Arc::new(Lease(count.clone())); + let (entered, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let (child_entered, child_running) = oneshot::channel(); + let (finish_child, child_permit) = mpsc::channel(); + let mut operation = Box::pin(with_operation_lifetime( + source, + parent.run(move || -> Result<()> { + let _ = entered.send(()); + let _ = permit.recv(); + let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + child_owner.dispatch(move || { + let tx = child_conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let committed: i64 = tx + .query_row("SELECT count(*) FROM writes", [], |row| row.get(0)) + .unwrap(); + assert_eq!(committed, 1, "child must see its parent's committed SQL"); + let _ = child_entered.send(()); + let _ = child_permit.recv(); + tx.execute("INSERT INTO writes VALUES (2)", []).unwrap(); + tx.commit().unwrap(); + })?; + Ok(()) + }), + )); + assert!(futures::poll!(&mut operation).is_pending()); + observed(running).await.unwrap(); + drop(operation); + release.send(()).unwrap(); + observed(child_running).await.unwrap(); + observed(parent.drain()).await.unwrap(); + assert_eq!( + count.load(std::sync::atomic::Ordering::SeqCst), + 1, + "accepted continuation lost authority when its parent finished" + ); + let mut drained = Box::pin(child.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + finish_child.send(()).unwrap(); + observed(drained).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); + } + + #[tokio::test] + async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let error = executor + .run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.execute("INSERT INTO writes VALUES (NULL)", [])?; + tx.commit()?; + Ok(()) + }) + .await + .unwrap_err(); + assert!(matches!(error, Error::Sqlite(_))); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test] + async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let error = executor + .run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + panic!("lost outcome after commit"); + }) + .await + .unwrap_err(); + assert!(matches!(error, Error::DbOperationOutcomeUnknown)); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42); + } + + #[tokio::test] + async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let error = with_operation_lifetime( + lease, + executor.run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + panic!("lost outcome before commit"); + }), + ) + .await + .unwrap_err(); + assert!(matches!(error, Error::DbOperationOutcomeUnknown)); + executor.drain().await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + struct DiscardedReplyHook { + expected_runtime: tokio::runtime::Id, + lease_count: Arc, + dropped: Option>, + } + + impl Drop for DiscardedReplyHook { + fn drop(&mut self) { + let runtime = + tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context"); + let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); + let same_runtime = runtime.id() == self.expected_runtime; + let dropped = self.dropped.take().unwrap(); + tokio::spawn(async move { + let _ = dropped.send((same_runtime, retained)); + }); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop() + { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut blocker = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut blocker).is_pending()); + observed(running).await.unwrap(); + let expected_runtime = tokio::runtime::Handle::current().id(); + let (hook, published) = oneshot::channel(); + let (finish, finishing) = mpsc::channel(); + let (dropped, reply_dropped) = oneshot::channel(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let operation_count = count.clone(); + let mut queued = Box::pin(with_operation_lifetime( + lease, + executor.run(move || -> Result<_> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + // Model synchronous Host hooks: publish only after commit, checking + // try_current and spawning without driving any future on this worker. + let runtime = tokio::runtime::Handle::try_current() + .expect("post-commit hook lost runtime context"); + assert_eq!(runtime.id(), expected_runtime); + tokio::spawn(async move { + let _ = hook.send(()); + }); + let _ = finishing.recv(); + Ok(DiscardedReplyHook { + expected_runtime, + lease_count: operation_count, + dropped: Some(dropped), + }) + }), + )); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + release.send(()).unwrap(); + observed(blocker).await.unwrap(); + observed(published).await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + finish.send(()).unwrap(); + assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1)); + observed(drain).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() { + let executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(2).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let mut releases = Vec::new(); + // Observe each worker executing before submitting the next operation, + // so queue occupancy and overload are deterministic, not timing guesses. + for _ in 0..2 { + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut operation = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut operation).is_pending()); + observed(running).await.unwrap(); + drop(operation); + releases.push(release); + } + let (finished, mut queued_finished) = oneshot::channel(); + let mut queued = Box::pin(executor.run(move || -> Result<()> { + let _ = finished.send(()); + Ok(()) + })); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + for _ in 0..8 { + let rejected_runs = rejected_runs.clone(); + assert!(matches!( + executor + .run(move || { + rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok::<_, Error>(()) + }) + .await, + Err(Error::DbCapacityExceeded) + )); + } + assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(matches!( + queued_finished.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + for release in releases { + release.send(()).unwrap(); + } + observed(drain).await.unwrap(); + observed(queued_finished).await.unwrap(); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[derive(Default)] + struct UnknownDiagnostic { + code: Option, + phase: Option, + accepted_job_id: Option, + } + + impl tracing::field::Visit for UnknownDiagnostic { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + match field.name() { + "code" => self.code = Some(value.to_owned()), + "phase" => self.phase = Some(value.to_owned()), + _ => {} + } + } + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + if field.name() == "accepted_job_id" { + self.accepted_job_id = Some(value); + } + } + fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} + } + + #[derive(Clone, Default)] + struct DiagnosticCapture(Arc>>); + + impl tracing_subscriber::Layer for DiagnosticCapture { + fn on_event( + &self, + event: &tracing::Event<'_>, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut diagnostic = UnknownDiagnostic::default(); + event.record(&mut diagnostic); + self.0.lock().unwrap().push(diagnostic); + } + } + + struct DispatchCleanup { + expected_runtime: tokio::runtime::Id, + lease_count: Arc, + started: Option>, + release: mpsc::Receiver<()>, + } + + impl Drop for DispatchCleanup { + fn drop(&mut self) { + // No panicking assertions during unwind: report observations to the + // test, and gate cleanup so drain/lifetime behavior is observable. + let same_runtime = tokio::runtime::Handle::try_current() + .is_ok_and(|runtime| runtime.id() == self.expected_runtime); + let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); + if let Some(started) = self.started.take() { + let _ = started.send((same_runtime, retained)); + } + let _ = self.release.recv(); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry() + { + use tracing_subscriber::layer::SubscriberExt; + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut blocker = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut blocker).is_pending()); + observed(running).await.unwrap(); + drop(blocker); // Cancellation does not free its running slot. + let expected_runtime = tokio::runtime::Handle::current().id(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let operation_count = count.clone(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let operation_attempts = attempts.clone(); + let (cleanup_started, cleaning_up) = oneshot::channel(); + let (finish, finishing) = mpsc::channel(); + // Model multiple tracing contexts without setting a global subscriber. + // This also prevents tracing's single-subscriber callsite optimization + // from caching an unsubscribed parallel test's Interest::never here. + let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry()); + let captured = DiagnosticCapture::default(); + let diagnostics = + tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone())); + with_operation_lifetime(lease, async { + tracing::dispatcher::with_default(&diagnostics, || { + executor.dispatch(move || { + let _cleanup = DispatchCleanup { + expected_runtime, + lease_count: operation_count, + started: Some(cleanup_started), + release: finishing, + }; + operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let tx = worker + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + tx.execute("INSERT INTO writes VALUES (1)", []).unwrap(); + tx.commit().unwrap(); + panic!("dispatched post-commit outcome lost"); + }) + }) + }) + .await + .unwrap(); // Dispatch is admitted synchronously; no waiter survives. + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let rejected_operation = rejected_runs.clone(); + assert!(matches!( + executor.dispatch(move || { + rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }), + Err(Error::DbCapacityExceeded) + )); + release.send(()).unwrap(); + assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1)); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + finish.send(()).unwrap(); + observed(drain).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); + { + let events = captured.0.lock().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!( + events[0].code.as_deref(), + Some("db_operation_outcome_unknown") + ); + assert_eq!(events[0].phase.as_deref(), Some("operation")); + assert!(events[0].accepted_job_id.is_some()); + } + let (completed, completion) = oneshot::channel(); + executor + .dispatch(move || { + let _ = completed.send(()); + }) + .unwrap(); + observed(executor.drain()).await.unwrap(); + observed(completion).await.unwrap(); + } +} diff --git a/crates/workspace-server/src/frozen_content_state_tests.rs b/crates/workspace-server/src/frozen_content_state_tests.rs index 138b83683..954d42ccf 100644 --- a/crates/workspace-server/src/frozen_content_state_tests.rs +++ b/crates/workspace-server/src/frozen_content_state_tests.rs @@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store @@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store diff --git a/crates/workspace-server/src/frozen_repository_key_tests.rs b/crates/workspace-server/src/frozen_repository_key_tests.rs index adeef98fe..798f1575d 100644 --- a/crates/workspace-server/src/frozen_repository_key_tests.rs +++ b/crates/workspace-server/src/frozen_repository_key_tests.rs @@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }; let migrated = store @@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let archived = store @@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = @@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() { // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = diff --git a/crates/workspace-server/src/frozen_runtime_binding_tests.rs b/crates/workspace-server/src/frozen_runtime_binding_tests.rs index fcf7dcc97..9c8bef71f 100644 --- a/crates/workspace-server/src/frozen_runtime_binding_tests.rs +++ b/crates/workspace-server/src/frozen_runtime_binding_tests.rs @@ -533,6 +533,7 @@ mod existing_workspace_provisioning { // This fixture contains only the schema-87 cutover authorities, not the // unrelated full Server schema. Exercise the real service/store methods. Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }) } @@ -644,6 +645,7 @@ mod existing_workspace_provisioning { drop(service); drop(store); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())), }); let service = WorkspaceSigningIdentityService::new( @@ -816,7 +818,7 @@ mod current_authority { use super::*; fn binding_store() -> (SqliteWorkspaceStore, WorkspaceRuntimeBinding) { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-key").unwrap(); store.with_conn(|conn| { diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index c0fe34c99..429ece684 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -23,7 +23,10 @@ use std::{ }; use workdir::{ Workdir, WorkdirError, WorkdirSessionHandle, - http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization}, + http::{ + OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization, + WorkdirHttpAuthorizationFuture, + }, }; use worker_runtime::RuntimeWorkspaceScope; use worker_runtime::catalog::{ @@ -1294,8 +1297,8 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { fn observation_source( &self, _worker_id: &str, - ) -> Option { - None + ) -> Result, RuntimeDiagnostic> { + Ok(None) } fn send_input(&self, worker_id: &str, _request: WorkerInputRequest) -> WorkerInputResult { @@ -2233,6 +2236,11 @@ impl RuntimeRegistry { let runtime = self.runtime(runtime_id)?; runtime .observation_source(worker_id) + .map_err(|diagnostic| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: runtime_id.to_owned(), + code: diagnostic.code, + message: diagnostic.message, + })? .ok_or_else(|| RuntimeRegistryError::UnknownWorker { worker: worker.clone(), }) @@ -3200,17 +3208,21 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { fn observation_source( &self, worker_id: &str, - ) -> Option { - let worker_ref = self.worker_ref(worker_id)?; + ) -> Result, RuntimeDiagnostic> { + let Some(worker_ref) = self.worker_ref(worker_id) else { + return Ok(None); + }; if self.runtime.worker_detail(&worker_ref).is_err() { - return None; + return Ok(None); } - Some(crate::observation::RuntimeObservationSource::embedded( - crate::observation::EmbeddedRuntimeObservationSource { - worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), - runtime: self.runtime.clone(), - worker_ref, - }, + Ok(Some( + crate::observation::RuntimeObservationSource::embedded( + crate::observation::EmbeddedRuntimeObservationSource { + worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), + runtime: self.runtime.clone(), + worker_ref, + }, + ), )) } @@ -3477,6 +3489,38 @@ impl WorkspaceRuntimeAuthorization { } } + /// Offload the complete live-binding check and request signing operation. + /// Claims are minted inside the bounded job, not cached before admission. + pub(crate) async fn issue_async( + &self, + method: String, + path_and_query: String, + operation: String, + worker_id: Option, + body: Vec, + ) -> Result { + let executor = self.store.db_executor(); + let authorization = self.clone(); + executor + .run(move || { + Ok::<_, Error>(authorization.issue( + &method, + &path_and_query, + &operation, + worker_id.as_deref(), + &body, + )) + }) + .await + .map_err(|error| { + diagnostic( + "workspace_runtime_authorization_unavailable", + HostDiagnosticSeverity::Error, + error.to_string(), + ) + })? + } + pub(crate) fn issue( &self, method: &str, @@ -3649,25 +3693,24 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization { impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization { fn bearer_token( &self, - method: &str, - path_and_query: &str, - body: &[u8], - ) -> Result { - if let Some(authorization) = &self.workspace_authorization { - return authorization - .issue( - method, - path_and_query, - workspace_runtime_operation(method, path_and_query), - None, - body, + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let authorization = self.clone(); + Box::pin(async move { + if let Some(authorizer) = &authorization.workspace_authorization { + let operation = workspace_runtime_operation(&method, &path_and_query).to_string(); + return authorizer + .issue_async(method, path_and_query, operation, None, body) + .await + .map_err(|error| WorkdirError::Unavailable(error.message)); + } + authorization.fallback_bearer_token.ok_or_else(|| { + WorkdirError::Unavailable( + "remote Runtime does not have bearer authorization configured".to_string(), ) - .map_err(|error| WorkdirError::Unavailable(error.message)); - } - self.fallback_bearer_token.clone().ok_or_else(|| { - WorkdirError::Unavailable( - "remote Runtime does not have bearer authorization configured".to_string(), - ) + }) }) } } @@ -5290,20 +5333,22 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { fn observation_source( &self, worker_id: &str, - ) -> Option { + ) -> Result, RuntimeDiagnostic> { let path = format!("/v1/workers/{worker_id}/protocol/ws"); let bearer_token = match &self.workspace_authorization { - Some(authorization) => authorization - .issue("GET", &path, "workers:protocol", Some(worker_id), &[]) - .ok(), + Some(authorization) => { + Some(authorization.issue("GET", &path, "workers:protocol", Some(worker_id), &[])?) + } None => self.bearer_token.clone(), }; - Some(crate::observation::RuntimeObservationSource::remote_ws( - crate::observation::RuntimeObservationSourceConfig { - worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), - endpoint: self.ws_endpoint(worker_id), - bearer_token, - }, + Ok(Some( + crate::observation::RuntimeObservationSource::remote_ws( + crate::observation::RuntimeObservationSourceConfig { + worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), + endpoint: self.ws_endpoint(worker_id), + bearer_token, + }, + ), )) } @@ -6348,6 +6393,79 @@ mod tests { use std::sync::{Arc, Mutex}; use std::thread; + #[tokio::test] + async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() { + use crate::db_executor::{DbExecutor, DbExecutorLimits}; + use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding}; + use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore; + + let executor = DbExecutor::new(DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::in_memory_for_test() + .unwrap() + .with_db_executor(executor.clone()), + ); + let service = WorkspaceSigningIdentityService::new( + store.clone(), + Arc::new(InMemoryWorkspaceSigningMaterialStore::default()), + ); + let authorization = RemoteWorkdirAuthorization { + workspace_authorization: Some(WorkspaceRuntimeAuthorization::new( + store, + service, + "https://backend.test", + WorkspaceRuntimeBinding { + workspace_id: "workspace".into(), + runtime_id: "runtime".into(), + display_name: "Runtime".into(), + base_url: "https://runtime.test".into(), + public_key_fingerprint: "unused".into(), + public_key: "unused".into(), + binding_id: "binding".into(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: "1".into(), + updated_at: "1".into(), + revoked_at: None, + }, + )), + fallback_bearer_token: Some("must-not-be-used".into()), + }; + + // Hold the running slot and fill the only queue slot. The callback must + // reject admission, not run inline or silently switch authentication. + let (release, wait) = std::sync::mpsc::channel::<()>(); + let (started, running) = tokio::sync::oneshot::channel(); + executor + .dispatch(move || { + let _ = started.send(()); + let _ = wait.recv_timeout(Duration::from_secs(5)); + }) + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), running) + .await + .unwrap() + .unwrap(); + executor.dispatch(|| {}).unwrap(); + let error = authorization + .bearer_token( + "POST".into(), + "/v1/workdir-sessions/s/operations".into(), + b"{}".to_vec(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string() + )); + drop(release); + executor.drain().await.unwrap(); + } + #[test] fn ticket_worker_spawn_intent_uses_worker_wire_role() { let intent = WorkerSpawnIntent::TicketRole { diff --git a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs index f68964f35..29c0cb52c 100644 --- a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs +++ b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs @@ -32,7 +32,8 @@ struct Fixture { impl Fixture { async fn new() -> Self { let temp = tempfile::tempdir().unwrap(); - let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap()); + let store = + Arc::new(SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap()); store .upsert_account(&AccountRecord { account_id: "owner".into(), diff --git a/crates/workspace-server/src/legacy_evidence_column_tests.rs b/crates/workspace-server/src/legacy_evidence_column_tests.rs index a59f2a0ca..33994b292 100644 --- a/crates/workspace-server/src/legacy_evidence_column_tests.rs +++ b/crates/workspace-server/src/legacy_evidence_column_tests.rs @@ -45,7 +45,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() { let artifacts = evidence_rows(&conn, "artifacts"); let receipts = evidence_rows(&conn, "repository_secret_legacy_receipts"); drop(conn); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); upgraded .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -73,7 +73,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() { Ok(()) }) .unwrap(); - for store in [upgraded, SqliteWorkspaceStore::in_memory().unwrap()] { + for store in [upgraded, SqliteWorkspaceStore::in_memory_for_test().unwrap()] { store.with_conn(|conn| { let mut stmt = conn.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")?; let objects = stmt.query_map([],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?)))?.collect::>>()?; diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index fefe24d92..eb113b079 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -10,6 +10,7 @@ pub mod backend_job; pub mod companion; pub mod config; pub mod config_source; +pub mod db_executor; pub mod feature_storage; pub mod hosts; pub mod memory_backend; @@ -159,6 +160,12 @@ pub enum Error { WorkspaceSigningIdentity { code: String, message: String }, #[error("Workspace Drive error: {0}")] Drive(#[from] workspace_drive::Error), + #[error("DB execution capacity exceeded")] + DbCapacityExceeded, + #[error("DB executor unavailable")] + DbExecutorUnavailable, + #[error("DB operation outcome unknown; do not automatically retry")] + DbOperationOutcomeUnknown, #[error("store error: {0}")] Store(String), } diff --git a/crates/workspace-server/src/memory_staging.rs b/crates/workspace-server/src/memory_staging.rs index ff8caadf7..631d1b33e 100644 --- a/crates/workspace-server/src/memory_staging.rs +++ b/crates/workspace-server/src/memory_staging.rs @@ -217,7 +217,7 @@ mod tests { async fn authority() -> (TempDir, SqliteWorkspaceAuthority) { let temp = TempDir::new().unwrap(); let db_path = temp.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), diff --git a/crates/workspace-server/src/migration_entrypoint_tests.rs b/crates/workspace-server/src/migration_entrypoint_tests.rs index d0eb2375d..41329a730 100644 --- a/crates/workspace-server/src/migration_entrypoint_tests.rs +++ b/crates/workspace-server/src/migration_entrypoint_tests.rs @@ -32,7 +32,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() { drop(conn); for attempt in 0..3 { let error = match attempt { - 0 => SqliteWorkspaceStore::open(&path).err().unwrap(), + 0 => SqliteWorkspaceStore::open_for_test(&path).err().unwrap(), 1 => SqliteWorkspaceStore::migration_plan(&path).unwrap_err(), _ => SqliteWorkspaceStore::migrate_database(&path).unwrap_err(), }; @@ -62,7 +62,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() { ) .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); assert_eq!(conn.query_row("SELECT input_fingerprint FROM worker_removal_operations WHERE operation_id='removal'",[],|r|r.get::<_,String>(0))?,"legacy-receipt"); @@ -109,7 +109,7 @@ fn migration_dry_run_and_apply_share_the_original_repository_secret_authority() assert!(!table_exists(&conn, "server_secret_objects").unwrap()); drop(conn); assert_eq!(SqliteWorkspaceStore::migrate_database(&path).unwrap(), plan); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?,LATEST_SCHEMA_VERSION); let (nonce, mut sealed): (Vec,Vec) = conn.query_row( diff --git a/crates/workspace-server/src/repository_access.rs b/crates/workspace-server/src/repository_access.rs index 304909848..c8a13c3f4 100644 --- a/crates/workspace-server/src/repository_access.rs +++ b/crates/workspace-server/src/repository_access.rs @@ -2290,7 +2290,7 @@ mod tests { ) { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); let timestamp = now(); for workspace_id in ["workspace-a", "workspace-b"] { futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord { diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index aa6ca692d..4aa310885 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -1116,7 +1116,7 @@ mod tests { WorkerId::from_legacy_u64(1) } fn setup() -> SqliteWorkspaceStore { - let s = SqliteWorkspaceStore::in_memory().unwrap(); + let s = SqliteWorkspaceStore::in_memory_for_test().unwrap(); s.with_conn(|c| { c.execute( "INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) \ diff --git a/crates/workspace-server/src/runtime_subscription.rs b/crates/workspace-server/src/runtime_subscription.rs index 3a7fdd0bd..998786c38 100644 --- a/crates/workspace-server/src/runtime_subscription.rs +++ b/crates/workspace-server/src/runtime_subscription.rs @@ -965,7 +965,7 @@ async fn connect_runtime( let mut request = endpoint .into_client_request() .map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?; - if let Some(token) = runtime_token(config, workspace_id)? { + if let Some(token) = runtime_token(config, workspace_id).await? { request.headers_mut().insert( "authorization", format!("Bearer {token}") @@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String { format!("{base}/v1/protocol/ws") } } -fn runtime_token( +async fn runtime_token( config: &RemoteRuntimeConfig, _workspace_id: &str, ) -> Result, String> { if let Some(authorization) = config.workspace_authorization.as_ref() { return authorization - .issue("GET", "/v1/protocol/ws", "workers:list", None, &[]) + .issue_async( + "GET".to_string(), + "/v1/protocol/ws".to_string(), + "workers:list".to_string(), + None, + Vec::new(), + ) + .await .map(Some) .map_err(|error| error.message); } diff --git a/crates/workspace-server/src/schema_equivalence_tests.rs b/crates/workspace-server/src/schema_equivalence_tests.rs index 37f5e218a..4063a675b 100644 --- a/crates/workspace-server/src/schema_equivalence_tests.rs +++ b/crates/workspace-server/src/schema_equivalence_tests.rs @@ -38,8 +38,8 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("upgraded.db"); prepare_retained_schema(&path, 85); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let mut mismatches = Vec::new(); for table in [ "workspace_tags", diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 5ed22bcc8..f782dfa9d 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -885,7 +885,7 @@ impl ExternalProviderWorkdirSession { } } - fn audit_operation( + async fn audit_operation( &self, operation_id: &ExternalWorkdirOperationId, operation_kind: &str, @@ -894,25 +894,34 @@ impl ExternalProviderWorkdirSession { let Some(audit) = &self.audit else { return Ok(()); }; + let audit = audit.clone(); + let operation_id = operation_id.clone(); + let operation_kind = operation_kind.to_owned(); + let outcome = outcome.to_owned(); + let connection = self.connection.clone(); audit .store - .record_external_workdir_audit( - &audit.workspace_id, - "worker", - &audit.actor_key, - "external_workdir_operation", - &self.connection.grant_id, - Some(operation_id.as_str()), - outcome, - &format!( - "workdir_id={} attachment_alias={} operation_kind={} generation={}", - self.connection.workdir_id, - audit.attachment_alias, - operation_kind, - self.connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) + .db_executor() + .run(move || { + audit.store.record_external_workdir_audit( + &audit.workspace_id, + "worker", + &audit.actor_key, + "external_workdir_operation", + &connection.grant_id, + Some(operation_id.as_str()), + &outcome, + &format!( + "workdir_id={} attachment_alias={} operation_kind={} generation={}", + connection.workdir_id, + audit.attachment_alias, + operation_kind, + connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await .map_err(|error| { workdir::WorkdirError::Unavailable(format!( "External Workdir operation audit failed: {error}" @@ -960,7 +969,8 @@ impl ExternalProviderWorkdirSession { } let operation_id = ExternalWorkdirOperationId::new(format!("op-{}", Uuid::now_v7())) .map_err(workdir::WorkdirError::InvalidArgument)?; - self.audit_operation(&operation_id, operation_kind, "requested")?; + self.audit_operation(&operation_id, operation_kind, "requested") + .await?; let (response, receive) = tokio::sync::oneshot::channel(); let mut cancel_guard = ExternalOperationCancelGuard { sender: self.connection.sender.clone(), @@ -977,7 +987,8 @@ impl ExternalProviderWorkdirSession { .await .is_err() { - self.audit_operation(&operation_id, operation_kind, "provider_unavailable")?; + self.audit_operation(&operation_id, operation_kind, "provider_unavailable") + .await?; return Err(workdir::WorkdirError::Unavailable( "External Workdir provider disconnected".to_string(), )); @@ -1007,7 +1018,8 @@ impl ExternalProviderWorkdirSession { } else { "failed" }, - )?; + ) + .await?; result } @@ -1577,6 +1589,7 @@ fn orchestrator_attention_state( #[derive(Clone)] pub struct WorkspaceApi { + host_executors: HostExecutors, pub(crate) config: ServerConfig, pub(crate) store: Arc, config_store: Arc, @@ -1675,6 +1688,7 @@ impl ServerApiContractService { struct WorkerRemovalService { workspace_id: String, store: Arc, + host_executors: HostExecutors, runtime: Weak, workdir_sessions: Arc>, workdir_session_locks: Arc>>>>, @@ -1688,6 +1702,7 @@ impl WorkerRemovalService { Self { workspace_id: api.config.workspace_id.clone(), store: api.store.clone(), + host_executors: api.host_executors.clone(), runtime: Arc::downgrade(&api.runtime), workdir_sessions: api.workdir_sessions.clone(), workdir_session_locks: api.workdir_session_locks.clone(), @@ -1853,6 +1868,30 @@ impl WorkerRemovalService { target_runtime_id: &str, target_worker_id: &str, reason: &str, + ) -> std::result::Result { + let service = self.clone(); + let runtime_id = target_runtime_id.to_owned(); + let worker_id = target_worker_id.to_owned(); + let reason = reason.to_owned(); + run_host_workflow( + self.host_executors.host().map_err(|e| e.to_string())?, + async move { + service + .execute_async_on_host(source, &runtime_id, &worker_id, &reason) + .await + .map_err(Error::Store) + }, + ) + .await + .map_err(|error| error.to_string()) + } + + async fn execute_async_on_host( + &self, + source: crate::worker_source::VerifiedWorkerMutationSource, + target_runtime_id: &str, + target_worker_id: &str, + reason: &str, ) -> std::result::Result { let reason = reason.trim(); if reason.is_empty() || reason.len() > 512 { @@ -1924,11 +1963,39 @@ impl WorkerRemovalService { )); } - self.execute_target_removal(&runtime, &target, reason, None) + self.execute_target_removal_on_host(&runtime, &target, reason, None) .await } async fn execute_target_removal( + &self, + _runtime: &RuntimeRegistry, + target: &RuntimeWorkerRef, + reason: &str, + parent_workspace_deletion_operation_id: Option<&str>, + ) -> std::result::Result { + let runtime = self + .runtime + .upgrade() + .ok_or_else(|| "Workspace Runtime registry is unavailable".to_string())?; + let service = self.clone(); + let target = target.clone(); + let reason = reason.to_owned(); + let parent = parent_workspace_deletion_operation_id.map(str::to_owned); + run_host_workflow( + self.host_executors.host().map_err(|e| e.to_string())?, + async move { + service + .execute_target_removal_on_host(&runtime, &target, &reason, parent.as_deref()) + .await + .map_err(Error::Store) + }, + ) + .await + .map_err(|error| error.to_string()) + } + + async fn execute_target_removal_on_host( &self, runtime: &RuntimeRegistry, target: &RuntimeWorkerRef, @@ -2177,6 +2244,7 @@ use workspace_admission::WorkspaceAdmission; #[derive(Clone)] pub struct WorkspaceServerApi { + host_executors: HostExecutors, template: Arc, store: Arc, catalog: WorkspaceCatalogService, @@ -2207,6 +2275,7 @@ impl WorkspaceServerApi { workspace_signing_material_root(&template.database_path), )); Self { + host_executors: HostExecutors::shared(), template: Arc::new(template), catalog: WorkspaceCatalogService::new(store.clone(), signing_materials.clone()), signing_materials, @@ -2222,6 +2291,23 @@ impl WorkspaceServerApi { } } + #[cfg(test)] + fn new_for_test(template: ServerConfig, store: Arc) -> Self { + let mut api = Self::new(template, store); + api.host_executors = HostExecutors::default(); + api + } + + // Own the complete synchronous store stage; callers retain their async + // lifecycle guards across it and never split a transaction around await. + async fn execute_store_stage( + &self, + operation: impl FnOnce(Arc) -> Result + Send + 'static, + ) -> Result { + let store = self.store.clone(); + self.store.db_executor().run(move || operation(store)).await + } + async fn admission_for_workspace(&self, workspace_id: &str) -> Arc { workspace_admission_gate(&self.admission_gates, workspace_id).await } @@ -2234,11 +2320,22 @@ impl WorkspaceServerApi { let Some(workspace) = self.store.get_workspace(workspace_id).await? else { return Ok(None); }; - let repositories = self.store.list_repositories(workspace_id)?; + let store = self.store.clone(); + let id = workspace_id.to_owned(); + let repositories = self + .store + .db_executor() + .run(move || store.list_repositories(&id)) + .await?; let config = self .template .for_catalog_workspace(&workspace, repositories)?; - let api = WorkspaceApi::new(config, self.store.clone()).await?; + let api = WorkspaceApi::new_with_host_executors( + config, + self.store.clone(), + self.host_executors.clone(), + ) + .await?; apis.insert(workspace_id.to_string(), api.clone()); Ok(Some(api)) } @@ -2251,34 +2348,46 @@ impl WorkspaceServerApi { drop(running); let api = self.clone(); tokio::spawn(async move { - if api.execute_workspace_deletion(&operation_id).await.is_err() { - let operation = api - .store - .workspace_deletion_operation_for_recovery(&operation_id) - .ok() - .flatten(); - let child_operation_ids = operation - .as_ref() - .map(|operation| operation.child_operation_ids.as_slice()) - .unwrap_or_default(); - let blockers = operation - .as_ref() - .map(|operation| operation.blockers.as_slice()) - .unwrap_or_default(); - let _ = api.store.update_workspace_deletion_operation( - &operation_id, - WorkspaceDeletionState::Failed, - child_operation_ids, - blockers, - Some("workspace_deletion_execution_failed"), - ); + if let Err(error) = api.execute_workspace_deletion(&operation_id).await { + tracing::warn!(%error, %operation_id, "Workspace deletion execution did not complete"); + if !matches!(error, Error::DbOperationOutcomeUnknown) { + let id = operation_id.clone(); + let _ = api + .execute_store_stage(move |store| { + let operation = store + .workspace_deletion_operation_for_recovery(&id) + .ok() + .flatten(); + let child_operation_ids = operation + .as_ref() + .map(|operation| operation.child_operation_ids.as_slice()) + .unwrap_or_default(); + let blockers = operation + .as_ref() + .map(|operation| operation.blockers.as_slice()) + .unwrap_or_default(); + store + .update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Failed, + child_operation_ids, + blockers, + Some("workspace_deletion_execution_failed"), + ) + .map(drop) + }) + .await; + } } api.running_deletions.lock().await.remove(&operation_id); }); } async fn recover_workspace_deletions(&self) -> Result<()> { - for operation_id in self.store.resumable_workspace_deletion_operation_ids()? { + let operations = self + .execute_store_stage(|store| store.resumable_workspace_deletion_operation_ids()) + .await?; + for operation_id in operations { self.schedule_workspace_deletion(operation_id).await; } Ok(()) @@ -2289,16 +2398,20 @@ impl WorkspaceServerApi { actor_account_id: &str, workspace_id: &str, ) -> Result { + let actor = actor_account_id.to_owned(); + let id = workspace_id.to_owned(); let mut preflight = self - .store - .workspace_deletion_preflight(actor_account_id, workspace_id)?; + .execute_store_stage(move |store| store.workspace_deletion_preflight(&actor, &id)) + .await?; let Some(api) = self.api_for_workspace(workspace_id).await? else { return Err(Error::InvalidInput("Workspace does not exist".to_string())); }; - for registry_worker in self - .store - .list_worker_registry(workspace_id, i64::MAX as usize)? - { + let id = workspace_id.to_owned(); + let workers = self + .execute_store_stage(move |store| store.list_worker_registry(&id, i64::MAX as usize)) + .await?; + api.host_executors.host()?.run(move || -> Result<_> { + for registry_worker in workers { let worker_key = registry_worker.display_name; match api.runtime.worker(®istry_worker.worker) { Ok(worker) if worker.state == "stopped" => {} @@ -2321,23 +2434,30 @@ impl WorkspaceServerApi { crate::workspace_deletion::bound_workspace_deletion_blockers(&mut preflight.blockers); preflight.can_delete = preflight.blockers.is_empty(); Ok(preflight) + }).await } async fn execute_workspace_deletion( &self, operation_id: &str, ) -> Result { - let prior = self - .store - .workspace_deletion_operation_for_recovery(operation_id)? - .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; - let operation = self.store.update_workspace_deletion_operation( - operation_id, - WorkspaceDeletionState::Running, - &prior.child_operation_ids, - &[], - None, - )?; + let id = operation_id.to_owned(); + let operation = self + .execute_store_stage(move |store| { + let prior = store + .workspace_deletion_operation_for_recovery(&id)? + .ok_or_else(|| { + Error::InvalidInput("Workspace deletion operation".to_string()) + })?; + store.update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Running, + &prior.child_operation_ids, + &[], + None, + ) + }) + .await?; let api = self .api_for_workspace(&operation.workspace_id) .await? @@ -2346,10 +2466,13 @@ impl WorkspaceServerApi { api.fence_drive().await?; let mut child_operation_ids = operation.child_operation_ids.clone(); let mut blockers = Vec::new(); - for worker in self - .store - .list_worker_registry(&operation.workspace_id, i64::MAX as usize)? - { + let workspace_id = operation.workspace_id.clone(); + let workers = self + .execute_store_stage(move |store| { + store.list_worker_registry(&workspace_id, i64::MAX as usize) + }) + .await?; + for worker in workers { let worker_key = worker.display_name.clone(); let target = worker.worker; let response = WorkerRemovalService::new(&api) @@ -2361,13 +2484,17 @@ impl WorkspaceServerApi { ) .await .map_err(Error::Store)?; + let id = operation_id.to_owned(); child_operation_ids = self - .store - .workspace_deletion_operation_for_recovery(operation_id)? - .ok_or_else(|| { - Error::Store("Workspace deletion operation disappeared".to_string()) - })? - .child_operation_ids; + .execute_store_stage(move |store| { + store + .workspace_deletion_operation_for_recovery(&id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + }) + .map(|operation| operation.child_operation_ids) + }) + .await?; if response.status != 200 { blockers.push(WorkspaceDeletionBlocker { kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, @@ -2379,22 +2506,36 @@ impl WorkspaceServerApi { } if blockers.is_empty() { - for workdir in self - .store - .list_workdir_registry(&operation.workspace_id, i64::MAX as usize)? - { - match execute_workdir_removal_for_workspace_deletion( - &api, - &workdir.workdir_id, - operation_id, - ) { + let workspace_id = operation.workspace_id.clone(); + let workdirs = self + .execute_store_stage(move |store| { + store.list_workdir_registry(&workspace_id, i64::MAX as usize) + }) + .await?; + for workdir in workdirs { + let removal_api = api.clone(); + let workdir_id = workdir.workdir_id.clone(); + let parent_id = operation_id.to_owned(); + match api + .host_executors + .host()? + .run(move || { + execute_workdir_removal_for_workspace_deletion( + &removal_api, + &workdir_id, + &parent_id, + ) + }) + .await + { Ok(child) => { + let id = operation_id.to_owned(); + let child_id = child.operation_id.clone(); child_operation_ids = self - .store - .append_workspace_deletion_child_operation( - operation_id, - &child.operation_id, - )? + .execute_store_stage(move |store| { + store.append_workspace_deletion_child_operation(&id, &child_id) + }) + .await? .child_operation_ids; if child.state != WorkdirRemovalOperationState::Completed || child.disposition != Some(WorkdirRemovalDisposition::Removed) @@ -2430,29 +2571,40 @@ impl WorkspaceServerApi { } if !blockers.is_empty() { - return self.store.update_workspace_deletion_operation( - operation_id, - WorkspaceDeletionState::Blocked, - &child_operation_ids, - &blockers, - None, - ); + let id = operation_id.to_owned(); + return self + .execute_store_stage(move |store| { + store.update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Blocked, + &child_operation_ids, + &blockers, + None, + ) + }) + .await; } api.purge_drive().await?; let feature_storage = api.feature_storage.clone(); - tokio::task::spawn_blocking(move || feature_storage.delete()) - .await - .map_err(|error| { - Error::Store(format!("Feature storage deletion task failed: {error}")) - })? - .map_err(|error| { - Error::Store(format!( - "failed to delete Workspace Feature storage: {error}" - )) - })?; - WorkspaceSigningIdentityService::new(self.store.clone(), self.signing_materials.clone()) - .delete_material(&operation.workspace_id)?; - let completed = self.store.finalize_workspace_deletion(operation_id)?; + api.store + .db_executor() + .run(move || { + feature_storage.delete().map_err(|error| { + Error::Store(format!( + "failed to delete Workspace Feature storage: {error}" + )) + }) + }) + .await?; + let materials = self.signing_materials.clone(); + let id = operation_id.to_owned(); + let completed = self + .execute_store_stage(move |store| { + WorkspaceSigningIdentityService::new(store.clone(), materials) + .delete_material(&operation.workspace_id)?; + store.finalize_workspace_deletion(&id) + }) + .await?; self.routers.lock().await.remove(&completed.workspace_id); if let Some(handle) = self .hook_handles @@ -2499,15 +2651,19 @@ impl WorkspaceServerApi { async fn shutdown_feature_storage(&self) -> Result<()> { let apis = self.apis.lock().await.values().cloned().collect::>(); for api in apis { - api.feature_storage.shutdown().map_err(|error| { - Error::Store(format!("failed to shut down Feature storage: {error}")) - })?; + shutdown_workspace_feature_storage(&api).await?; } Ok(()) } async fn preload(&self) -> Result<()> { - for workspace in self.store.list_workspaces()? { + let store = self.store.clone(); + let workspaces = self + .store + .db_executor() + .run(move || store.list_workspaces()) + .await?; + for workspace in workspaces { let _ = self .router_for_workspace(&workspace.workspace_id) .await? @@ -2577,7 +2733,7 @@ async fn resolve_server_actor( headers: &HeaderMap, ) -> std::result::Result, Error> { let cookie_name = auth_public_config(api.template.as_ref()).cookie_name; - resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await + resolve_request_actor(api.store.clone(), headers, &cookie_name).await } fn signed_request_target(uri: &Uri) -> &str { @@ -2616,6 +2772,11 @@ fn runtime_request_proof_rejection( workspace_id: &str, error: &crate::worker_source::RuntimeRequestProofError, ) -> Response { + let status = if error.is_availability_error() { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::UNAUTHORIZED + }; // Never format an error or raw claims. IDs are fixed-length fingerprints; // verification labels describe proof-check progress, not authorization. let path = path.split('?').next().unwrap_or(path); @@ -2627,7 +2788,7 @@ fn runtime_request_proof_rejection( target: "yoi::auth", event = "runtime_request_proof_rejected", method, path = log_path, workspace_id, request_id, - status = 401, + status = status.as_u16(), reason = error.reason, proof_verification = error.verification, claimed_runtime_id_hash = error.runtime_id_hash.as_deref(), @@ -2642,8 +2803,12 @@ fn runtime_request_proof_rejection( ); repository_api_rejection( path, - StatusCode::UNAUTHORIZED, - "invalid runtime request proof", + status, + if error.is_availability_error() { + "Runtime authentication authority unavailable" + } else { + "invalid runtime request proof" + }, ) } @@ -2718,7 +2883,7 @@ async fn authorize_scoped_workspace_request( worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION }; let source = crate::worker_source::verify_runtime_request_source_proof_with_clock( - api.store.as_ref(), + api.store.clone(), api.template.as_ref(), &proof, workspace_id, @@ -2883,41 +3048,38 @@ async fn authorize_workspace_api_request( if !matches!( *request.method(), Method::GET | Method::HEAD | Method::OPTIONS - ) && !api - .store - .get_workspace(&workspace_id) - .await - .ok() - .flatten() - .is_some_and(|workspace| workspace.state == "active") - { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); + ) { + let workspace = match api.store.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(error) => return server_error_response(error), + }; + if !workspace.is_some_and(|workspace| workspace.state == "active") { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); + } } return next.run(request).await; } let AuthConfig::Passkey { cookie_name, .. } = &api.config.auth; - let actor = match crate::auth::resolve_request_actor( - api.store.as_ref(), - request.headers(), - cookie_name, - ) - .await - { - Ok(Some(actor)) => actor, - Ok(None) if public_server_api => return next.run(request).await, - Ok(None) | Err(_) => { - return repository_api_rejection( - &request_path, - StatusCode::UNAUTHORIZED, - "authentication required", - ); - } - }; + let actor = + match crate::auth::resolve_request_actor(api.store.clone(), request.headers(), cookie_name) + .await + { + Ok(Some(actor)) => actor, + Ok(None) if public_server_api => return next.run(request).await, + Err(error) => return server_error_response(error), + Ok(None) => { + return repository_api_rejection( + &request_path, + StatusCode::UNAUTHORIZED, + "authentication required", + ); + } + }; let cookie_authenticated = matches!(actor.auth_method, ActorAuthMethod::BrowserSession); let mutating = !matches!( *request.method(), @@ -2940,20 +3102,18 @@ async fn authorize_workspace_api_request( } } request.extensions_mut().insert(actor); - if mutating - && !api - .store - .get_workspace(&workspace_id) - .await - .ok() - .flatten() - .is_some_and(|workspace| workspace.state == "active") - { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); + if mutating { + let workspace = match api.store.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(error) => return server_error_response(error), + }; + if !workspace.is_some_and(|workspace| workspace.state == "active") { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); + } } next.run(request).await } @@ -3004,7 +3164,7 @@ fn workspace_request_is_callback(method: &Method, path: &str, workspace_id: &str async fn dispatch_workspace_request( State(api): State, - mut request: Request, + request: Request, ) -> Response { let path = request.uri().path().to_owned(); let workspace_id = scoped_workspace_id(&path); @@ -3030,6 +3190,23 @@ async fn dispatch_workspace_request( } else { None }; + if let Some(lease) = _request_lease { + crate::db_executor::with_operation_lifetime( + Arc::new(lease), + dispatch_admitted_workspace_request(api, request), + ) + .await + } else { + dispatch_admitted_workspace_request(api, request).await + } +} + +async fn dispatch_admitted_workspace_request( + api: WorkspaceServerApi, + mut request: Request, +) -> Response { + let path = request.uri().path().to_owned(); + let workspace_id = scoped_workspace_id(&path); if let Some(workspace_id) = workspace_id && (path.starts_with("/api/w/") || path.starts_with("/api/runtime/v1/workspaces/") @@ -3046,7 +3223,13 @@ async fn dispatch_workspace_request( Err(error) => return server_error_response(error), } } else { - let workspaces = match api.store.list_workspaces() { + let store = api.store.clone(); + let workspaces = match api + .store + .db_executor() + .run(move || store.list_workspaces()) + .await + { Ok(workspaces) => workspaces, Err(error) => return server_error_response(error), }; @@ -3322,6 +3505,19 @@ impl WorkspaceApi { } pub async fn new(config: ServerConfig, store: Arc) -> Result { + Self::new_with_host_executors(config, store, HostExecutors::shared()).await + } + + #[cfg(test)] + async fn new_for_test(config: ServerConfig, store: Arc) -> Result { + Self::new_with_host_executors(config, store, HostExecutors::default()).await + } + + async fn new_with_host_executors( + config: ServerConfig, + store: Arc, + host_executors: HostExecutors, + ) -> Result { let resource_broker = BackendResourceBroker::default(); if store.get_workspace(&config.workspace_id).await?.is_none() { return Err(crate::Error::Config(format!( @@ -3398,6 +3594,7 @@ impl WorkspaceApi { store, Arc::new(execution_backend), resource_broker, + host_executors, Some(worker_remove_dispatcher), ) .await?; @@ -3458,6 +3655,7 @@ impl WorkspaceApi { store, execution_backend, BackendResourceBroker::default(), + HostExecutors::default(), None, ) .await @@ -3468,6 +3666,7 @@ impl WorkspaceApi { store: Arc, execution_backend: Arc, resource_broker: BackendResourceBroker, + host_executors: HostExecutors, worker_remove_dispatcher: Option< Arc, >, @@ -3478,8 +3677,18 @@ impl WorkspaceApi { config.workspace_id ))); } - import_configured_repositories(store.as_ref(), &config)?; - config.repositories = load_configured_repositories_from_store(store.as_ref(), &config)?; + let repository_store = store.clone(); + let repository_config = config.clone(); + config.repositories = store + .db_executor() + .run(move || { + import_configured_repositories(repository_store.as_ref(), &repository_config)?; + load_configured_repositories_from_store( + repository_store.as_ref(), + &repository_config, + ) + }) + .await?; let embedded_runtime = EmbeddedWorkerRuntime::new_fs_store_with_execution_backend( config.workspace_id.clone(), config.embedded_runtime_store_root.clone(), @@ -3494,20 +3703,34 @@ impl WorkspaceApi { RuntimeSubscriptionBroker::new(config.workspace_id.clone()); runtime_subscription_broker .register_embedded_runtime(embedded_runtime_id, embedded_subscription_runtime); - let config_store = Arc::new(crate::SqliteWorkspaceStore::open( - config.database_path.clone(), - )?); - let repository_secrets = Arc::new(RepositorySecretService::open( - config_store.clone(), - &config.database_path, - )?); + let database_path = config.database_path.clone(); + let executor = store.db_executor(); + let config_executor = executor.clone(); + let (config_store, repository_secrets) = executor + .run(move || { + let config_store = Arc::new( + crate::SqliteWorkspaceStore::open(database_path.clone())? + .with_db_executor(config_executor), + ); + let repository_secrets = Arc::new(RepositorySecretService::open( + config_store.clone(), + &database_path, + )?); + Ok::<_, Error>((config_store, repository_secrets)) + }) + .await?; let signing_materials: Arc = Arc::new(FsWorkspaceSigningMaterialStore::new( workspace_signing_material_root(&config.database_path), )); let signing_identities = WorkspaceSigningIdentityService::new(store.clone(), signing_materials); - signing_identities.get_validated(&config.workspace_id)?; + let identities = signing_identities.clone(); + let workspace_id = config.workspace_id.clone(); + store + .db_executor() + .run(move || identities.get_validated(&workspace_id)) + .await?; let backend_url = config .backend_base_url .clone() @@ -3550,11 +3773,19 @@ impl WorkspaceApi { .with_provider(Arc::new(RuntimeConfigSchemaProvider)) .with_provider(Arc::new(RepositoryAccessConfigSchemaProvider)) .with_provider(Arc::new(skills::SkillConfigSchemaProvider)); - config_store.ensure_workspace_config_materialized_with_schema( - &config.workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), - config_schema_registry.compose()?, - )?; + let materialization_store = config_store.clone(); + let workspace_id = config.workspace_id.clone(); + let schema = config_schema_registry.compose()?; + store + .db_executor() + .run(move || { + materialization_store.ensure_workspace_config_materialized_with_schema( + &workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + schema, + ) + }) + .await?; let worker_projection = crate::worker_projection::WorkerProjectionService::new( config.workspace_id.clone(), store.clone(), @@ -3565,18 +3796,28 @@ impl WorkspaceApi { .await .map_err(|error| Error::InvalidInput(error.to_string()))?; } - let feature_storage = crate::FeatureStorage::for_server_database(&config.database_path) - .and_then(|storage| storage.workspace(&config.workspace_id)) - .map_err(|error| { - Error::Store(format!("failed to initialize Feature storage: {error}")) - })?; - let subjektiv_registration = crate::subjektiv::SubjektivStore::register(&feature_storage) - .map_err(|error| { - Error::Store(format!("failed to register subjektiv storage: {error}")) - })?; + let database_path = config.database_path.clone(); + let workspace_id = config.workspace_id.clone(); + let (feature_storage, subjektiv_registration, authority) = store + .db_executor() + .run(move || { + let feature_storage = crate::FeatureStorage::for_server_database(&database_path) + .and_then(|storage| storage.workspace(&workspace_id)) + .map_err(|error| { + Error::Store(format!("failed to initialize Feature storage: {error}")) + })?; + let registration = crate::subjektiv::SubjektivStore::register(&feature_storage) + .map_err(|error| { + Error::Store(format!("failed to register subjektiv storage: {error}")) + })?; + let authority = SqliteWorkspaceAuthority::new(database_path, workspace_id)?; + Ok::<_, Error>((feature_storage, registration, authority)) + }) + .await?; let drive_host = Arc::new(crate::workspace_drive_host::WorkspaceDriveHost::new(&feature_storage).await?); let mut api = Self { + host_executors, drive_host, config_store, repository_secrets, @@ -3586,10 +3827,7 @@ impl WorkspaceApi { crate::prompt_settings::WorkspacePromptProjectionCache::default(), feature_storage, subjektiv_registration, - authority: SqliteWorkspaceAuthority::new( - config.database_path.clone(), - config.workspace_id.clone(), - )?, + authority, _worker_projection_shutdown: Arc::new(WorkerProjectionShutdownGuard { service: Arc::downgrade(&worker_projection), }), @@ -3639,62 +3877,88 @@ impl WorkspaceApi { )) .map_err(|message| Error::Config(message.to_string()))?; } - let expired_external_workdirs = api.store.reconcile_external_workdir_grants_after_restart( - &api.config.workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; + let recovery_api = api.clone(); + let expired_external_workdirs = api + .store + .db_executor() + .run(move || { + recovery_api + .store + .reconcile_external_workdir_grants_after_restart( + &recovery_api.config.workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await?; for workdir_id in expired_external_workdirs { cleanup_external_workdir_attachments(&api, &workdir_id) .await .map_err(|error| error.error)?; } - for grant in api + let grant_api = api.clone(); + let grants = api .store - .list_external_workdir_grants(&api.config.workspace_id)? + .db_executor() + .run(move || { + grant_api + .store + .list_external_workdir_grants(&grant_api.config.workspace_id) + }) + .await?; + for grant in grants .into_iter() .filter(|grant| !matches!(grant.status.as_str(), "revoked" | "expired")) { schedule_external_workdir_expiry(&api, &grant)?; } - recover_workdir_removals(&api)?; + let workdir_recovery_api = api.clone(); + api.host_executors + .host()? + .run(move || recover_workdir_removals(&workdir_recovery_api)) + .await?; recover_runtime_removals(&api).await; - let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let deliveries = api - .store - .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; - for delivery in deliveries { - let Some(job) = api - .store - .get_backend_job(&api.config.workspace_id, &delivery.job_id)? - else { - continue; - }; - let Some(attempt) = api.store.get_backend_job_attempt( - &api.config.workspace_id, - &delivery.job_id, - &delivery.attempt_id, - )? - else { - continue; - }; - if job.state == BackendJobState::Completed - && attempt.state == BackendJobAttemptState::Completed - { - api.deliver_backend_job_result( - &BackendJobResultAcceptance { - job, - attempt, - replayed: true, - }, - &delivery.target, - ); - } - } - api.recover_backend_jobs()?; - if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { - dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); - } - Ok(api) + api.host_executors + .host()? + .run(move || { + let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let deliveries = api + .store + .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; + for delivery in deliveries { + let Some(job) = api + .store + .get_backend_job(&api.config.workspace_id, &delivery.job_id)? + else { + continue; + }; + let Some(attempt) = api.store.get_backend_job_attempt( + &api.config.workspace_id, + &delivery.job_id, + &delivery.attempt_id, + )? + else { + continue; + }; + if job.state == BackendJobState::Completed + && attempt.state == BackendJobAttemptState::Completed + { + api.deliver_backend_job_result( + &BackendJobResultAcceptance { + job, + attempt, + replayed: true, + }, + &delivery.target, + ); + } + } + api.recover_backend_jobs()?; + if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + Ok(api) + }) + .await } pub fn workspace_id(&self) -> &str { @@ -4087,77 +4351,44 @@ impl WorkspaceApi { { return false; } - let Ok(handle) = tokio::runtime::Handle::try_current() else { - tracing::warn!( - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "Backend Job Worker cleanup remains durable because no Tokio runtime is available" - ); + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + tracing::warn!(job_id = %attempt.job_id, "Backend Job cleanup remains durable without a runtime"); return false; }; - let (attempt, claimed) = match self.store.claim_backend_job_worker_cleanup( - &self.config.workspace_id, - &attempt.job_id, - &attempt.attempt_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { - Ok(claim) => claim, - Err(error) => { - tracing::warn!( - %error, - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "failed to claim durable Backend Job Worker cleanup" - ); - return false; - } - }; - if !claimed { - return false; - } - let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed - || attempt.failure_category.as_deref() == Some("invalid_result"); - if waits_for_run_teardown { - let api = self.clone(); - let cleanup_handle = handle.clone(); - handle.spawn(async move { - if let Err(error) = api.wait_for_backend_job_worker_run_teardown(&attempt).await { - tracing::warn!( - %error, - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "Backend Job Worker Run teardown fence failed" - ); - return; - } - api.spawn_claimed_backend_job_worker_cleanup(&cleanup_handle, attempt); - }); - } else { - self.spawn_claimed_backend_job_worker_cleanup(&handle, attempt); - } - true - } - - fn spawn_claimed_backend_job_worker_cleanup( - &self, - handle: &tokio::runtime::Handle, - attempt: BackendJobAttemptRecord, - ) { let api = self.clone(); let job_id = attempt.job_id.clone(); let attempt_id = attempt.attempt_id.clone(); - handle.spawn_blocking(move || { + // Admit before claiming: overload must leave durable Pending/Failed + // ownership recoverable, never strand a Running claim without a job. + let admission = self.host_executors.cleanup().and_then(|executor| { + executor.dispatch(move || { let cleanup = || -> Result<()> { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| Error::Store(error.to_string()))? - .block_on(api.cleanup_backend_job_attempt_worker(attempt)) + let (attempt, claimed) = api.store.claim_backend_job_worker_cleanup( + &api.config.workspace_id, &job_id, &attempt_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; + if !claimed { return Ok(()); } + let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed + || attempt.failure_category.as_deref() == Some("invalid_result"); + runtime.block_on(async { + if waits_for_run_teardown { + api.wait_for_backend_job_worker_run_teardown(&attempt).await?; + } + // Already on the Host owner; removal must not submit to and + // wait on its own lane. All transaction helpers return before I/O. + api.cleanup_backend_job_attempt_worker(attempt).await + }) }; if let Err(error) = cleanup() { tracing::warn!(%error, %job_id, %attempt_id, "Backend Job Worker cleanup failed"); } + }) }); + if let Err(error) = admission { + tracing::warn!(%error, "Backend Job cleanup was not admitted; its durable ownership is unchanged"); + return false; + } + true } async fn wait_for_backend_job_worker_run_teardown( @@ -4292,7 +4523,7 @@ impl WorkspaceApi { // recover that midpoint; without such an operation this remains // a diagnosable retryable failure rather than a direct DB delete. let response = WorkerRemovalService::new(self) - .execute_target_removal( + .execute_target_removal_on_host( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4358,7 +4589,7 @@ impl WorkspaceApi { } let response = WorkerRemovalService::new(self) - .execute_target_removal( + .execute_target_removal_on_host( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4485,6 +4716,7 @@ impl WorkspaceApi { let attempt_id = attempt_id.to_string(); handle.spawn(async move { tokio::time::sleep(delay).await; + let admission = api.host_executors.host().and_then(|executor| executor.dispatch(move || { let Ok(Some(attempt)) = api.store .get_backend_job_attempt(&api.config.workspace_id, &job_id, &attempt_id) @@ -4508,6 +4740,10 @@ impl WorkspaceApi { { api.drain_reserved_backend_jobs(); } + })); + if let Err(error) = admission { + tracing::warn!(%error, "Backend Job timeout was not admitted; its deadline remains durable"); + } }); } @@ -6225,7 +6461,7 @@ async fn attach_server_request_context( } else { let auth = service.auth_api(); match resolve_request_actor( - auth.store.as_ref(), + auth.store.clone(), request.headers(), &auth_public_config(&auth.config).cookie_name, ) @@ -7191,7 +7427,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyRegistrationComplete, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = complete_passkey_registration(&self.auth_api(), request) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || complete_passkey_registration(&api, request)) + .await .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyRegistrationComplete::Status200 { @@ -7222,7 +7463,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyLoginComplete, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = complete_passkey_login(&self.auth_api(), request) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || complete_passkey_login(&api, request)) + .await .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyLoginComplete::Status200 { @@ -7255,7 +7501,11 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - approve_device_login(&self.auth_api(), actor, request) + let api = self.auth_api(); + api.store + .db_executor() + .run(move || approve_device_login(&api, actor, request)) + .await .map_err(ApiError::into_repository_api_error) } @@ -7285,7 +7535,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthLogout, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = logout_browser_session(&self.auth_api(), cookie) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || logout_browser_session(&api, cookie)) + .await .map_err(ApiError::into_repository_api_error)?; Ok(server_api::server_api_responses::AuthLogout::Status200 { body, @@ -7300,9 +7555,12 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.server_api()?; let Some(actor) = context.actor else { + let catalog = api.catalog.clone(); return if api - .catalog - .is_empty() + .store + .db_executor() + .run(move || catalog.is_empty()) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())? { Ok(WorkspaceCatalogListResponse(Vec::new())) @@ -7315,12 +7573,17 @@ impl server_api::ServerApi for ServerApiContractService { )) }; }; + let catalog = api.catalog.clone(); let workspaces = api - .catalog - .list( - &actor.account_id, - query.limit.unwrap_or(100).try_into().unwrap_or(200), - ) + .store + .db_executor() + .run(move || { + catalog.list( + &actor.account_id, + query.limit.unwrap_or(100).try_into().unwrap_or(200), + ) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; Ok(WorkspaceCatalogListResponse( workspaces.into_iter().map(workspace_summary).collect(), @@ -7341,9 +7604,12 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; + let catalog = api.catalog.clone(); let created = api - .catalog - .create(request, actor.account_id) + .store + .db_executor() + .run(move || catalog.create(request, actor.account_id)) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; api.router_for_workspace(&created.workspace.workspace_id) .await @@ -7388,10 +7654,14 @@ impl server_api::ServerApi for ServerApiContractService { ) })?; let admission_for_workspace = api.admission_for_workspace(&workspace_id).await; - let _mutation_guard = admission_for_workspace.lock_deletion().await; + let mutation_guard = Arc::new(admission_for_workspace.lock_deletion().await); + let account_id = actor.account_id.clone(); + let operation_id = request.operation_id.clone(); let existing = api - .store - .workspace_deletion_operation(&actor.account_id, &request.operation_id) + .execute_store_stage(move |store| { + store.workspace_deletion_operation(&account_id, &operation_id) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; if existing.is_none() { let preflight = api @@ -7407,9 +7677,18 @@ impl server_api::ServerApi for ServerApiContractService { )); } } + let account_id = actor.account_id.clone(); + let id = workspace_id.clone(); + let reserve_request = request.clone(); + let reserve_guard = mutation_guard.clone(); let reservation = api - .store - .reserve_workspace_deletion(&actor.account_id, &workspace_id, &request) + .execute_store_stage(move |store| { + // Keep deletion closed until the accepted reservation really finishes, + // even when the HTTP task drops its original guard. + let _guard = reserve_guard; + store.reserve_workspace_deletion(&account_id, &id, &reserve_request) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; if let Some(handle) = api.hook_handles.lock().await.remove(&workspace_id) { handle.abort(); @@ -7449,17 +7728,19 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - api.store - .workspace_deletion_operation(&actor.account_id, &operation_id) - .map_err(|error| ApiError::from(error).into_repository_api_error())? - .ok_or_else(|| { - server_api::RepositoryApiError::new( - StatusCode::NOT_FOUND.as_u16(), - "Not Found", - "Workspace deletion operation not found", - Vec::new(), - ) - }) + api.execute_store_stage(move |store| { + store.workspace_deletion_operation(&actor.account_id, &operation_id) + }) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? + .ok_or_else(|| { + server_api::RepositoryApiError::new( + StatusCode::NOT_FOUND.as_u16(), + "Not Found", + "Workspace deletion operation not found", + Vec::new(), + ) + }) } async fn workspace_current( @@ -7494,13 +7775,20 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::WorkspaceRuntimeConfig, server_api::RepositoryApiError, > { - workspace_runtime_config_result( - self.workspace_api()?, - context, - &workspace_id, - query, - if_none_match, - ) + let api = self.workspace_api()?.clone(); + api.store + .db_executor() + .run(move || { + Ok::<_, Error>(workspace_runtime_config_result( + &api, + context, + &workspace_id, + query, + if_none_match, + )) + }) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? } async fn workspace_metadata_settings( @@ -8425,16 +8713,28 @@ impl server_api::ServerApi for ServerApiContractService { )); } let worker = RuntimeWorkerRef::new(runtime_id.clone(), worker_id.clone()); - let observation = api - .runtime - .worker_session(&workspace_id, &worker) - .map_err(|_error| { - runtime_api::RuntimeApiError::new( - StatusCode::BAD_GATEWAY.as_u16(), - "worker_session_observation_failed", - "Worker Session observation failed", + let operation_api = api.clone(); + let operation_workspace = workspace_id.clone(); + let runtime_result = api + .host_executors + .host() + .map_err(runtime_execution_error)? + .run(move || { + Ok::<_, Error>( + operation_api + .runtime + .worker_session(&operation_workspace, &worker), ) - })?; + }) + .await + .map_err(runtime_execution_error)?; + let observation = runtime_result.map_err(|_error| { + runtime_api::RuntimeApiError::new( + StatusCode::BAD_GATEWAY.as_u16(), + "worker_session_observation_failed", + "Worker Session observation failed", + ) + })?; Ok(server_api::WorkspaceWorkerSessionResponse { subject: server_api::WorkspaceWorkerSubject::RuntimeWorker { runtime_id, @@ -8469,16 +8769,29 @@ impl server_api::ServerApi for ServerApiContractService { )); } let worker = RuntimeWorkerRef::new(runtime_id.clone(), worker_id.clone()); - let observation = api - .runtime - .worker_session_history(&workspace_id, &worker, query.cursor, query.limit) - .map_err(|_error| { - runtime_api::RuntimeApiError::new( - StatusCode::BAD_GATEWAY.as_u16(), - "worker_session_history_failed", - "Worker Session history request failed", - ) - })?; + let operation_api = api.clone(); + let operation_workspace = workspace_id.clone(); + let runtime_result = api + .host_executors + .host() + .map_err(runtime_execution_error)? + .run(move || { + Ok::<_, Error>(operation_api.runtime.worker_session_history( + &operation_workspace, + &worker, + query.cursor, + query.limit, + )) + }) + .await + .map_err(runtime_execution_error)?; + let observation = runtime_result.map_err(|_error| { + runtime_api::RuntimeApiError::new( + StatusCode::BAD_GATEWAY.as_u16(), + "worker_session_history_failed", + "Worker Session history request failed", + ) + })?; Ok(server_api::WorkspaceWorkerSessionHistoryResponse { subject: server_api::WorkspaceWorkerSubject::RuntimeWorker { runtime_id, @@ -9120,62 +9433,70 @@ impl server_api::ServerApi for ServerApiContractService { "expected_binding_id must contain between 1 and 256 bytes", ))); } - let now = Utc::now().to_rfc3339(); - if let Err(error) = api - .store - .revoke_workspace_runtime_binding_key( - &workspace_id, - &runtime_id, - &request.expected_binding_id, - &actor.account_id, - &now, - ) - .await - { - let kind = match error { - Error::RuntimeBindingIdConflict { .. } => { - Some(server_api::RuntimeTrustConflictKind::StaleBinding) - } - Error::RuntimeBindingFingerprintConflict { .. } => { - Some(server_api::RuntimeTrustConflictKind::FingerprintInUse) - } - _ => None, - }; - if let Some(kind) = kind { - let current = api - .store - .get_workspace_runtime_binding(&workspace_id, &runtime_id) - .await - .map_err(|error| map_error(ApiError::from(error)))?; - return Err(server_api::RuntimeManagementApiError::trust_conflict( - server_api::RuntimeTrustConflictResponse { - error: kind, - message: match kind { - server_api::RuntimeTrustConflictKind::StaleBinding => { - "the Runtime trust binding changed; reload before retrying".to_string() - } - server_api::RuntimeTrustConflictKind::FingerprintInUse => { - "the public key is already bound to another Runtime in this Workspace".to_string() - } + // Own the whole revoke outcome through local publication and response. + // The accepted DB mutation outlives a cancelled HTTP waiter; its Host + // parent must also remove expectations and terminate subscriptions. + // SQL keeps its independent owner, and response reads stay on this Host + // workflow rather than attempting nested admission to the same lane. + run_host_workflow(api.host_executors.host()?, async move { + let now = Utc::now().to_rfc3339(); + if let Err(error) = api + .store + .revoke_workspace_runtime_binding_key( + &workspace_id, + &runtime_id, + &request.expected_binding_id, + &actor.account_id, + &now, + ) + .await + { + let kind = match error { + Error::RuntimeBindingIdConflict { .. } => { + Some(server_api::RuntimeTrustConflictKind::StaleBinding) + } + Error::RuntimeBindingFingerprintConflict { .. } => { + Some(server_api::RuntimeTrustConflictKind::FingerprintInUse) + } + _ => None, + }; + if let Some(kind) = kind { + let current = api + .store + .get_workspace_runtime_binding(&workspace_id, &runtime_id) + .await + .map_err(|error| map_error(ApiError::from(error)))?; + return Err(server_api::RuntimeManagementApiError::trust_conflict( + server_api::RuntimeTrustConflictResponse { + error: kind, + message: match kind { + server_api::RuntimeTrustConflictKind::StaleBinding => { + "the Runtime trust binding changed; reload before retrying".to_string() + } + server_api::RuntimeTrustConflictKind::FingerprintInUse => { + "the public key is already bound to another Runtime in this Workspace".to_string() + } + }, + current_binding_id: current.as_ref().map(|binding| binding.binding_id.clone()), + current_fingerprint: current + .as_ref() + .map(|binding| binding.public_key_fingerprint.clone()), }, - current_binding_id: current.as_ref().map(|binding| binding.binding_id.clone()), - current_fingerprint: current - .as_ref() - .map(|binding| binding.public_key_fingerprint.clone()), - }, - )); + )); + } + return Err(map_error(ApiError::from(error))); } - return Err(map_error(ApiError::from(error))); - } - api.runtime_binding_expectations - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&(workspace_id.clone(), runtime_id.clone())); - api.runtime_subscription_broker - .unregister_runtime(&runtime_id); - workspace_runtime_detail(&api, &workspace_id, &runtime_id) - .await - .map_err(map_error) + api.runtime_binding_expectations + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&(workspace_id.clone(), runtime_id.clone())); + api.runtime_subscription_broker + .unregister_runtime(&runtime_id); + workspace_runtime_detail_on_host(&api, &workspace_id, &runtime_id) + .await + .map_err(map_error) + }) + .await } async fn runtime_connection_test( @@ -9393,7 +9714,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; restore_runtime_worker_with_context(api, runtime_id, worker_id, operation_context, request) .await .map(|Json(response)| response) @@ -9411,7 +9732,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; scoped_restore_runtime_worker_with_context( @@ -9438,10 +9759,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, true) @@ -9458,10 +9780,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, false) @@ -9498,7 +9821,7 @@ impl server_api::ServerApi for ServerApiContractService { server_api::RepositoryApiError, > { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; execute_runtime_cleanup_with_context(&api, &runtime_id, request, &operation_context) @@ -9515,8 +9838,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -9537,10 +9861,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -9700,8 +10025,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -9723,10 +10049,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -9747,8 +10074,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -9770,10 +10098,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -10588,23 +10917,28 @@ impl server_api::ServerApi for ServerApiContractService { query: server_api::TicketAssignmentsQuery, ) -> std::result::Result { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; + let api = self.workspace_api()?.clone(); api.store - .page_ticket_assignments( - &workspace_id, - &ticket_id, - query.after_assignment_id.as_deref(), - query.limit.unwrap_or(30), - ) - .map(server_api::TicketAssignmentsPage) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error) + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + api.store + .page_ticket_assignments( + &workspace_id, + &ticket_id, + query.after_assignment_id.as_deref(), + query.limit.unwrap_or(30), + ) + .map(server_api::TicketAssignmentsPage) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error) + }) + .await } async fn ticket_assignments_page_browser( &self, @@ -10627,24 +10961,29 @@ impl server_api::ServerApi for ServerApiContractService { server_api::TicketAssignmentDiagnosticsPage, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; + let api = self.workspace_api()?.clone(); api.store - .page_ticket_assignment_diagnostics( - &workspace_id, - &ticket_id, - &query.assignment_id, - query.after_input_event_id.as_deref(), - query.after_start_operation_id.as_deref(), - query.limit.unwrap_or(30), - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error) + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + api.store + .page_ticket_assignment_diagnostics( + &workspace_id, + &ticket_id, + &query.assignment_id, + query.after_input_event_id.as_deref(), + query.after_start_operation_id.as_deref(), + query.limit.unwrap_or(30), + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error) + }) + .await } async fn ticket_assignment_diagnostics_browser( &self, @@ -10669,28 +11008,45 @@ impl server_api::ServerApi for ServerApiContractService { server_api::TicketAssignmentAnswerResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let (actor, source) = - authenticated_assignment_source(api, &contract_request_headers(&context)?, &context) + let api = self.workspace_api()?.clone(); + // Mixed live Runtime discovery, whole answer transaction, and committed-input + // dispatch share one accepted finite Host owner, including waiter cancellation. + // Neither Runtime I/O nor its completion waits on the sole DB worker. + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let (actor, source) = authenticated_assignment_source( + &api, + &contract_request_headers(&context)?, + &context, + ) .map_err(ApiError::from) .map_err(ApiError::into_repository_api_error)?; - let receipt = api - .store - .answer_ticket_assignment(&workspace_id, &ticket_id, &request.0, &source, &actor) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - // The entire event/audit/receipt transaction has committed. Response loss - // retries the saved receipt; delivery scan deduplicates accepted inputs. - if let Some(orchestrator) = find_workspace_orchestrator(api) { - dispatch_orchestrator_assignment_attention(api, &orchestrator.worker); - } - Ok(server_api::TicketAssignmentAnswerResponse(receipt)) + let receipt = api + .store + .answer_ticket_assignment( + &workspace_id, + &ticket_id, + &request.0, + &source, + &actor, + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + // The entire event/audit/receipt transaction has committed. Response loss + // retries the saved receipt; delivery scan deduplicates accepted inputs. + if let Some(orchestrator) = find_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + Ok(server_api::TicketAssignmentAnswerResponse(receipt)) + }) + .await } async fn ticket_assignment_answer_browser( &self, @@ -10714,23 +11070,28 @@ impl server_api::ServerApi for ServerApiContractService { query: server_api::TicketAssignmentTrackingQuery, ) -> std::result::Result { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let values = if let Some(assignment_id) = query.assignment_id { - api.store - .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) - .map(|value| value.into_iter().collect::>()) - } else { - api.authority.ticket_assignment_tracking(&ticket_id) - } - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - Ok(server_api::TicketAssignmentTrackingList(values)) + let api = self.workspace_api()?.clone(); + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let values = if let Some(assignment_id) = query.assignment_id { + api.store + .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) + .map(|value| value.into_iter().collect::>()) + } else { + api.authority.ticket_assignment_tracking(&ticket_id) + } + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + Ok(server_api::TicketAssignmentTrackingList(values)) + }) + .await } async fn ticket_assignment_tracking_update( @@ -11455,7 +11816,7 @@ impl server_api::ServerApi for ServerApiContractService { let api = self.workspace_api().map_err(|_| { workspace_config::pre_error(Error::Store("Workspace unavailable".into())) })?; - let worker = workspace_config::identity(api, &context, &workspace_id)?; + let worker = workspace_config::identity(api, &context, &workspace_id).await?; workspace_config::get(api, &worker) .await .map(server_api::WorkspaceConfigCurrentResponse) @@ -11472,7 +11833,7 @@ impl server_api::ServerApi for ServerApiContractService { let api = self.workspace_api().map_err(|_| { workspace_config::pre_error(Error::Store("Workspace unavailable".into())) })?; - let worker = workspace_config::identity(api, &context, &workspace_id)?; + let worker = workspace_config::identity(api, &context, &workspace_id).await?; workspace_config::attach(api, &worker, request).await } async fn current_worker_workspace_config_observe( @@ -11487,7 +11848,7 @@ impl server_api::ServerApi for ServerApiContractService { let api = self.workspace_api().map_err(|_| { workspace_config::pre_error(Error::Store("Workspace unavailable".into())) })?; - let worker = workspace_config::identity(api, &context, &workspace_id)?; + let worker = workspace_config::identity(api, &context, &workspace_id).await?; workspace_config::observe(api, &worker, request).await } async fn current_worker_workspace_config_read( @@ -11502,7 +11863,7 @@ impl server_api::ServerApi for ServerApiContractService { let api = self.workspace_api().map_err(|_| { workspace_config::pre_error(Error::Store("Workspace unavailable".into())) })?; - let worker = workspace_config::identity(api, &context, &workspace_id)?; + let worker = workspace_config::identity(api, &context, &workspace_id).await?; workspace_config::read(api, &worker, request).await } async fn current_worker_workspace_config_commit( @@ -11517,7 +11878,7 @@ impl server_api::ServerApi for ServerApiContractService { let api = self.workspace_api().map_err(|_| { workspace_config::pre_error(Error::Store("Workspace unavailable".into())) })?; - let worker = workspace_config::identity(api, &context, &workspace_id)?; + let worker = workspace_config::identity(api, &context, &workspace_id).await?; workspace_config::commit(api, &worker, request).await } async fn workspace_config_grant_create( @@ -11557,21 +11918,30 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirCatalogResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) + let api = self.workspace_api()?.clone(); + let headers = current_worker_contract_headers(&context)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host() + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &workspace_id, + &headers, + )?) + }) + .await .map_err(ApiError::into_repository_api_error)?; - let worker = current_worker_identity( - api, - &workspace_id, - ¤t_worker_contract_headers(&context)?, - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - { - let session_lock = current_worker_session_lock(api, &worker); - let _guard = session_lock.lock().await; - list_current_worker_workdir_catalog_for_worker(api, &worker, query) - } + let session_lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(session_lock.lock_owned().await); + run_worker_workdir_db_stage_locked(&api, &guard, move |api| { + list_current_worker_workdir_catalog_for_worker(&api, &worker, query) + }) + .await .map_err(ApiError::into_repository_api_error) } @@ -11584,20 +11954,31 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirAttachmentListResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) + let api = self.workspace_api()?.clone(); + let headers = current_worker_contract_headers(&context)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host() + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &workspace_id, + &headers, + )?) + }) + .await .map_err(ApiError::into_repository_api_error)?; - let worker = current_worker_identity( - api, - &workspace_id, - ¤t_worker_contract_headers(&context)?, - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let session_lock = current_worker_session_lock(api, &worker); - let _session_guard = session_lock.lock().await; - list_current_worker_workdir_attachments(api, &worker, query) - .map_err(ApiError::into_repository_api_error) + let session_lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(session_lock.lock_owned().await); + run_worker_workdir_db_stage_locked(&api, &guard, move |api| { + list_current_worker_workdir_attachments(&api, &worker, query) + }) + .await + .map_err(ApiError::into_repository_api_error) } async fn current_worker_workdir_attach( @@ -11810,6 +12191,16 @@ impl server_api::ServerApi for ServerApiContractService { } } +fn runtime_execution_error(error: Error) -> runtime_api::RuntimeApiError { + let code = match &error { + Error::DbCapacityExceeded => "db_capacity_exceeded", + Error::DbExecutorUnavailable => "db_executor_unavailable", + Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown", + _ => "backend_execution_failed", + }; + runtime_api::RuntimeApiError::new(api_error_status(&error).as_u16(), code, error.to_string()) +} + async fn scoped_worker_session_attachment( State(api): State, AxumPath((workspace_id, runtime_id, worker_id, session_id, attachment_id)): AxumPath<( @@ -11829,10 +12220,25 @@ async fn scoped_worker_session_attachment( .into_response(); } let worker = RuntimeWorkerRef::new(runtime_id, worker_id); - match api - .runtime - .worker_session_attachment(&workspace_id, &worker, &session_id, &attachment_id) - { + let executor = match api.host_executors.host() { + Ok(value) => value, + Err(error) => return ApiError::from(error).into_response(), + }; + let fetched = executor + .run(move || { + Ok::<_, Error>(api.runtime.worker_session_attachment( + &workspace_id, + &worker, + &session_id, + &attachment_id, + )) + }) + .await; + let fetched = match fetched { + Ok(value) => value, + Err(error) => return ApiError::from(error).into_response(), + }; + match fetched { Ok(attachment) => { let Ok(content_type) = HeaderValue::from_str(&attachment.media_type) else { return ( @@ -12073,15 +12479,28 @@ pub async fn serve( let orchestrator_hook = tokio::spawn(run_orchestrator_turn_end_hook(api.clone())); let result = axum::serve(listener, build_router(api.clone())).await; orchestrator_hook.abort(); - let storage_result = api - .feature_storage - .shutdown() - .map_err(|error| Error::Store(format!("failed to shut down Feature storage: {error}"))); + let _ = orchestrator_hook.await; + let storage_result = shutdown_workspace_feature_storage(&api).await; result?; storage_result?; Ok(()) } +// Feature shutdown fences new operations and waits for synchronous connection +// holders. Accepted close retains ownership after its async waiter disappears. +async fn shutdown_workspace_feature_storage(api: &WorkspaceApi) -> Result<()> { + api.worker_projection.shutdown_async().await; + let storage = api.feature_storage.clone(); + api.store + .db_executor() + .run(move || { + storage.shutdown().map_err(|error| { + Error::Store(format!("failed to shut down Feature storage: {error}")) + }) + }) + .await +} + #[derive(Debug, Serialize, Deserialize)] pub struct ProjectListResponse { pub workspace_id: String, @@ -12299,8 +12718,13 @@ async fn scoped_list_flows( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult>> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) + }) + .await } async fn scoped_put_flow( @@ -12308,28 +12732,34 @@ async fn scoped_put_flow( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let definition = flow::compile_flow_source(&request.content).map_err(|error| { - Error::InvalidInput(format!( - "invalid Flow source: {}", - serde_json::to_string(&error.diagnostics) - .unwrap_or_else(|_| "diagnostics unavailable".to_string()) - )) - })?; - let expected_path = format!("flows/{}.dcdl", definition.name); - if request.path != expected_path { - return Err( - Error::InvalidInput(format!("Flow source path must be `{expected_path}`")).into(), - ); - } - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); - Ok(Json(api.store.put_flow_source_for_kind( - &path.workspace_id, - FlowSourceKind::Workspace, - &request.path, - &request.content, - &now, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let definition = flow::compile_flow_source(&request.content).map_err(|error| { + Error::InvalidInput(format!( + "invalid Flow source: {}", + serde_json::to_string(&error.diagnostics) + .unwrap_or_else(|_| "diagnostics unavailable".to_string()) + )) + })?; + let expected_path = format!("flows/{}.dcdl", definition.name); + if request.path != expected_path { + return Err(Error::InvalidInput(format!( + "Flow source path must be `{expected_path}`" + )) + .into()); + } + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + Ok(Json(api.store.put_flow_source_for_kind( + &path.workspace_id, + FlowSourceKind::Workspace, + &request.path, + &request.content, + &now, + )?)) + }) + .await } async fn scoped_resolve_flow_source( @@ -12337,65 +12767,79 @@ async fn scoped_resolve_flow_source( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = match &request.selector { - flow::FlowSelector::Builtin { slug } => { - let builtin = flow::builtin_flow_source(slug) - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let definition = builtin.compile().map_err(|error| { - Error::Store(format!( - "compile built-in Flow {slug:?}: {:?}", - error.diagnostics - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: format!("builtin:{slug}"), - content_digest: definition.content_digest.clone(), - definition, - } - } - flow::FlowSelector::Workspace { slug } => { - let source = api - .store - .get_flow_source_by_name(&path.workspace_id, FlowSourceKind::Workspace, slug)? - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let content = api - .store - .get_flow_source_content( - &path.workspace_id, - &source.flow_id, - &source.content_digest, - )? - .ok_or_else(|| { - Error::Store(format!( - "resolved Flow content {}@{} is missing", - source.flow_id, source.content_digest - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: source.flow_id, - content_digest: content.content_digest, - definition: content.definition, - } - } - }; - Ok(Json(resolved)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let resolved = match &request.selector { + flow::FlowSelector::Builtin { slug } => { + let builtin = flow::builtin_flow_source(slug) + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let definition = builtin.compile().map_err(|error| { + Error::Store(format!( + "compile built-in Flow {slug:?}: {:?}", + error.diagnostics + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: format!("builtin:{slug}"), + content_digest: definition.content_digest.clone(), + definition, + } + } + flow::FlowSelector::Workspace { slug } => { + let source = api + .store + .get_flow_source_by_name( + &path.workspace_id, + FlowSourceKind::Workspace, + slug, + )? + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let content = api + .store + .get_flow_source_content( + &path.workspace_id, + &source.flow_id, + &source.content_digest, + )? + .ok_or_else(|| { + Error::Store(format!( + "resolved Flow content {}@{} is missing", + source.flow_id, source.content_digest + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: source.flow_id, + content_digest: content.content_digest, + definition: content.definition, + } + } + }; + Ok(Json(resolved)) + }) + .await } async fn scoped_get_flow( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = api - .store - .get_flow_source(&path.workspace_id, &path.flow_id)? - .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; - Ok(Json(source)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = api + .store + .get_flow_source(&path.workspace_id, &path.flow_id)? + .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; + Ok(Json(source)) + }) + .await } async fn scoped_get_workspace_settings( @@ -12466,8 +12910,13 @@ async fn scoped_get_workspace_signing_identity( "Workspace public identity access", ) .await?; - let identity = api.signing_identities.get_validated(&path.workspace_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) + api.store + .db_executor() + .run(move || { + let identity = api.signing_identities.get_validated(&path.workspace_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) + }) + .await } async fn scoped_provision_workspace_signing_identity( @@ -12482,10 +12931,15 @@ async fn scoped_provision_workspace_signing_identity( "Workspace signing identity provisioning", ) .await?; - let identity = api - .signing_identities - .provision_existing(&path.workspace_id, &actor.account_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) + api.store + .db_executor() + .run(move || { + let identity = api + .signing_identities + .provision_existing(&path.workspace_id, &actor.account_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) + }) + .await } fn project_workspace_signing_identity( @@ -12563,12 +13017,19 @@ async fn scoped_get_workspace_config_history( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let snapshot = api - .config_store - .load_workspace_config_history(&path.workspace_id, &path.content_digest)? - .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.content_digest.clone())))?; - Ok(Json(config_tree_snapshot_to_api(snapshot))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let snapshot = api + .config_store + .load_workspace_config_history(&path.workspace_id, &path.content_digest)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId(path.content_digest.clone())) + })?; + Ok(Json(config_tree_snapshot_to_api(snapshot))) + }) + .await } #[derive(Debug, Deserialize)] @@ -12581,34 +13042,44 @@ async fn scoped_get_prompt_projection( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidInput( - "Workspace config is not initialized".to_string(), - )) - })?; - let projection = api - .prompt_projection_cache - .resolve(&path.workspace_id, &state)?; - Ok(Json(projection.as_ref().clone())) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidInput( + "Workspace config is not initialized".to_string(), + )) + })?; + let projection = api + .prompt_projection_cache + .resolve(&path.workspace_id, &state)?; + Ok(Json(projection.as_ref().clone())) + }) + .await } async fn scoped_get_workspace_memory_settings( State(api): State, AxumPath(workspace_id): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .get_workspace_memory_settings(&workspace_id) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .get_workspace_memory_settings(&workspace_id) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) + }) + .await } async fn scoped_update_workspace_memory_settings( @@ -12616,19 +13087,24 @@ async fn scoped_update_workspace_memory_settings( AxumPath(workspace_id): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .update_workspace_memory_settings( - &workspace_id, - &request.expected_language, - &request.language, - ) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .update_workspace_memory_settings( + &workspace_id, + &request.expected_language, + &request.language, + ) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) + }) + .await } async fn require_workspace_owner( @@ -12682,10 +13158,15 @@ async fn scoped_get_repository_access_projection( Extension(actor): Extension, ) -> ApiResult> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - Ok(Json(active_repository_access_projection( - &api, - &path.workspace_id, - )?)) + api.store + .db_executor() + .run(move || { + Ok(Json(active_repository_access_projection( + &api, + &path.workspace_id, + )?)) + }) + .await } async fn scoped_list_repository_ssh_credentials( @@ -12694,13 +13175,18 @@ async fn scoped_list_repository_ssh_credentials( Extension(actor): Extension, ) -> ApiResult>> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - api.repository_secrets - .ensure_workspace_default_credential(&path.workspace_id)?; - let projection = active_repository_access_projection(&api, &path.workspace_id)?; - Ok(Json( - api.repository_secrets - .list_credentials(&path.workspace_id, &projection)?, - )) + api.store + .db_executor() + .run(move || { + api.repository_secrets + .ensure_workspace_default_credential(&path.workspace_id)?; + let projection = active_repository_access_projection(&api, &path.workspace_id)?; + Ok(Json( + api.repository_secrets + .list_credentials(&path.workspace_id, &projection)?, + )) + }) + .await } async fn scoped_get_repository_ssh_credential( @@ -12709,14 +13195,19 @@ async fn scoped_get_repository_ssh_credential( Extension(actor): Extension, ) -> ApiResult> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - let projection = active_repository_access_projection(&api, &path.workspace_id)?; - let credential = api - .repository_secrets - .list_credentials(&path.workspace_id, &projection)? - .into_iter() - .find(|credential| credential.credential_id == path.credential_id) - .ok_or_else(|| Error::InvalidRecordId(path.credential_id.clone()))?; - Ok(Json(credential)) + api.store + .db_executor() + .run(move || { + let projection = active_repository_access_projection(&api, &path.workspace_id)?; + let credential = api + .repository_secrets + .list_credentials(&path.workspace_id, &projection)? + .into_iter() + .find(|credential| credential.credential_id == path.credential_id) + .ok_or_else(|| Error::InvalidRecordId(path.credential_id.clone()))?; + Ok(Json(credential)) + }) + .await } async fn scoped_get_repository_ssh_public_key( @@ -12725,11 +13216,16 @@ async fn scoped_get_repository_ssh_public_key( Extension(actor): Extension, ) -> ApiResult> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - let public_key = api - .repository_secrets - .credential_public_key(&path.workspace_id, &path.credential_id)? - .ok_or_else(|| Error::InvalidRecordId(path.credential_id.clone()))?; - Ok(Json(public_key)) + api.store + .db_executor() + .run(move || { + let public_key = api + .repository_secrets + .credential_public_key(&path.workspace_id, &path.credential_id)? + .ok_or_else(|| Error::InvalidRecordId(path.credential_id.clone()))?; + Ok(Json(public_key)) + }) + .await } async fn scoped_create_repository_ssh_credential( @@ -12739,18 +13235,25 @@ async fn scoped_create_repository_ssh_credential( Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - if request.credential_id - == crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID - { - return Err(Error::InvalidInput( - "Workspace default SSH credential id is reserved".to_string(), - ) - .into()); - } - let credential = - api.repository_secrets - .create_credential(&path.workspace_id, request, &actor.account_id)?; - Ok((StatusCode::CREATED, Json(credential))) + api.store + .db_executor() + .run(move || { + if request.credential_id + == crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID + { + return Err(Error::InvalidInput( + "Workspace default SSH credential id is reserved".to_string(), + ) + .into()); + } + let credential = api.repository_secrets.create_credential( + &path.workspace_id, + request, + &actor.account_id, + )?; + Ok((StatusCode::CREATED, Json(credential))) + }) + .await } async fn scoped_generate_repository_ssh_credential( @@ -12760,20 +13263,25 @@ async fn scoped_generate_repository_ssh_credential( Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - if request.credential_id - == crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID - { - return Err(Error::InvalidInput( - "Workspace default SSH credential id is reserved".to_string(), - ) - .into()); - } - let credential = api.repository_secrets.generate_credential( - &path.workspace_id, - request, - &actor.account_id, - )?; - Ok((StatusCode::CREATED, Json(credential))) + api.store + .db_executor() + .run(move || { + if request.credential_id + == crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID + { + return Err(Error::InvalidInput( + "Workspace default SSH credential id is reserved".to_string(), + ) + .into()); + } + let credential = api.repository_secrets.generate_credential( + &path.workspace_id, + request, + &actor.account_id, + )?; + Ok((StatusCode::CREATED, Json(credential))) + }) + .await } async fn scoped_rotate_repository_ssh_credential( @@ -12783,12 +13291,17 @@ async fn scoped_rotate_repository_ssh_credential( Json(request): Json, ) -> ApiResult> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - Ok(Json(api.repository_secrets.rotate_credential( - &path.workspace_id, - &path.credential_id, - request, - &actor.account_id, - )?)) + api.store + .db_executor() + .run(move || { + Ok(Json(api.repository_secrets.rotate_credential( + &path.workspace_id, + &path.credential_id, + request, + &actor.account_id, + )?)) + }) + .await } async fn scoped_delete_repository_ssh_credential( @@ -12798,15 +13311,20 @@ async fn scoped_delete_repository_ssh_credential( Json(request): Json, ) -> ApiResult { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - let projection = active_repository_access_projection(&api, &path.workspace_id)?; - api.repository_secrets.delete_credential( - &path.workspace_id, - &path.credential_id, - request, - &actor.account_id, - &projection, - )?; - Ok(StatusCode::NO_CONTENT) + api.store + .db_executor() + .run(move || { + let projection = active_repository_access_projection(&api, &path.workspace_id)?; + api.repository_secrets.delete_credential( + &path.workspace_id, + &path.credential_id, + request, + &actor.account_id, + &projection, + )?; + Ok(StatusCode::NO_CONTENT) + }) + .await } async fn scoped_list_repository_ssh_host_trusts( @@ -12815,11 +13333,16 @@ async fn scoped_list_repository_ssh_host_trusts( Extension(actor): Extension, ) -> ApiResult>> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - let projection = active_repository_access_projection(&api, &path.workspace_id)?; - Ok(Json( - api.repository_secrets - .list_host_trusts(&path.workspace_id, &projection)?, - )) + api.store + .db_executor() + .run(move || { + let projection = active_repository_access_projection(&api, &path.workspace_id)?; + Ok(Json( + api.repository_secrets + .list_host_trusts(&path.workspace_id, &projection)?, + )) + }) + .await } async fn scoped_get_repository_ssh_host_trust( @@ -12828,14 +13351,19 @@ async fn scoped_get_repository_ssh_host_trust( Extension(actor): Extension, ) -> ApiResult> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - let projection = active_repository_access_projection(&api, &path.workspace_id)?; - let host_trust = api - .repository_secrets - .list_host_trusts(&path.workspace_id, &projection)? - .into_iter() - .find(|host_trust| host_trust.host_trust_id == path.host_trust_id) - .ok_or_else(|| Error::InvalidRecordId(path.host_trust_id.clone()))?; - Ok(Json(host_trust)) + api.store + .db_executor() + .run(move || { + let projection = active_repository_access_projection(&api, &path.workspace_id)?; + let host_trust = api + .repository_secrets + .list_host_trusts(&path.workspace_id, &projection)? + .into_iter() + .find(|host_trust| host_trust.host_trust_id == path.host_trust_id) + .ok_or_else(|| Error::InvalidRecordId(path.host_trust_id.clone()))?; + Ok(Json(host_trust)) + }) + .await } async fn scoped_put_repository_ssh_host_trust( @@ -12845,15 +13373,22 @@ async fn scoped_put_repository_ssh_host_trust( Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - let status = if request.expected_fingerprint.is_some() { - StatusCode::OK - } else { - StatusCode::CREATED - }; - let host_trust = - api.repository_secrets - .put_host_trust(&path.workspace_id, request, &actor.account_id)?; - Ok((status, Json(host_trust))) + api.store + .db_executor() + .run(move || { + let status = if request.expected_fingerprint.is_some() { + StatusCode::OK + } else { + StatusCode::CREATED + }; + let host_trust = api.repository_secrets.put_host_trust( + &path.workspace_id, + request, + &actor.account_id, + )?; + Ok((status, Json(host_trust))) + }) + .await } async fn scoped_delete_repository_ssh_host_trust( @@ -12863,15 +13398,20 @@ async fn scoped_delete_repository_ssh_host_trust( Json(request): Json, ) -> ApiResult { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; - let projection = active_repository_access_projection(&api, &path.workspace_id)?; - api.repository_secrets.delete_host_trust( - &path.workspace_id, - &path.host_trust_id, - request, - &actor.account_id, - &projection, - )?; - Ok(StatusCode::NO_CONTENT) + api.store + .db_executor() + .run(move || { + let projection = active_repository_access_projection(&api, &path.workspace_id)?; + api.repository_secrets.delete_host_trust( + &path.workspace_id, + &path.host_trust_id, + request, + &actor.account_id, + &projection, + )?; + Ok(StatusCode::NO_CONTENT) + }) + .await } fn config_content_type_to_api( @@ -13139,41 +13679,51 @@ async fn scoped_get_workspace_config_tree( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { - snapshot: ConfigTreeSnapshot::empty(), - contract: config_source::ToolchainContract::new( - config_source::DEFAULT_SCHEMA_VERSION, - Vec::new(), - config_source::DEFAULT_IMPORT_POLICY_VERSION, - ), - projection_digest: config_source::digest_bytes(b"[]"), - }); - Ok(Json(workspace_config_state_to_api(state))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { + snapshot: ConfigTreeSnapshot::empty(), + contract: config_source::ToolchainContract::new( + config_source::DEFAULT_SCHEMA_VERSION, + Vec::new(), + config_source::DEFAULT_IMPORT_POLICY_VERSION, + ), + projection_digest: config_source::digest_bytes(b"[]"), + }); + Ok(Json(workspace_config_state_to_api(state))) + }) + .await } async fn scoped_get_workspace_config_entry( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let virtual_path = config_source::VirtualPath::parse(&path.path) - .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - let entry = state - .snapshot - .get(&virtual_path) - .cloned() - .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; - Ok(Json(config_entry_to_api(entry))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let virtual_path = config_source::VirtualPath::parse(&path.path) + .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + let entry = state + .snapshot + .get(&virtual_path) + .cloned() + .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; + Ok(Json(config_entry_to_api(entry))) + }) + .await } async fn scoped_commit_workspace_config_tree( @@ -13181,13 +13731,18 @@ async fn scoped_commit_workspace_config_tree( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - let request = config_commit_request_from_api(request)?; - validate_workspace_scope(&api, &path.workspace_id)?; - let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; - Ok(( - StatusCode::CREATED, - Json(workspace_config_state_to_api(state)), - )) + api.host_executors + .host()? + .run(move || { + let request = config_commit_request_from_api(request)?; + validate_workspace_scope(&api, &path.workspace_id)?; + let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; + Ok(( + StatusCode::CREATED, + Json(workspace_config_state_to_api(state)), + )) + }) + .await } /// Shared canonical commit and activation boundary for UI and logical config operations. @@ -13251,20 +13806,25 @@ async fn scoped_get_profile_settings( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - Ok(Json( - crate::profile_settings::project_profiles_from_workspace_config( - &path.workspace_id, - &state, - )? - .settings, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + Ok(Json( + crate::profile_settings::project_profiles_from_workspace_config( + &path.workspace_id, + &state, + )? + .settings, + )) + }) + .await } async fn scoped_list_tickets( @@ -13280,8 +13840,13 @@ async fn scoped_get_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - browser_ticket_detail(&api, &path.id) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_query_tickets( @@ -13289,8 +13854,13 @@ async fn scoped_query_tickets( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_tickets(query)?)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_tickets(query)?)) + }) + .await } async fn scoped_show_ticket( @@ -13298,11 +13868,16 @@ async fn scoped_show_ticket( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let backend = browser_ticket_backend(&api)?; - Ok(Json(api.authority.read_ticket_detail_with_backend( - &path.id, query, &backend, - )?)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let backend = browser_ticket_backend(&api)?; + Ok(Json(api.authority.read_ticket_detail_with_backend( + &path.id, query, &backend, + )?)) + }) + .await } fn parse_ticket_assignment_role(role: &str) -> ApiResult { @@ -13378,19 +13953,24 @@ async fn scoped_list_ticket_assignments( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let ticket = api.authority.ticket(&path.id)?; - let assignments = api - .store - .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; - Ok(Json(server_api::TicketRoleAssignmentsResponse { - workspace_id: path.workspace_id, - ticket_id: ticket.id, - assignments: assignments - .into_iter() - .map(project_ticket_role_assignment_record) - .collect(), - })) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let ticket = api.authority.ticket(&path.id)?; + let assignments = api + .store + .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; + Ok(Json(server_api::TicketRoleAssignmentsResponse { + workspace_id: path.workspace_id, + ticket_id: ticket.id, + assignments: assignments + .into_iter() + .map(project_ticket_role_assignment_record) + .collect(), + })) + }) + .await } async fn compensate_manual_ticket_worker_binding( @@ -13529,6 +14109,31 @@ async fn start_manual_ticket_worker_assignment( operation_id: &str, selected: &[server_api::TicketWorkerAttachmentBinding], delegation: Option<&TicketStartDelegation>, +) -> ApiResult { + let api = api.clone(); + let record = record.clone(); + let operation_id = operation_id.to_owned(); + let selected = selected.to_vec(); + let delegation = delegation.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + start_manual_ticket_worker_assignment_on_host( + &api, + &record, + &operation_id, + &selected, + delegation.as_ref(), + ) + .await + }) + .await +} + +async fn start_manual_ticket_worker_assignment_on_host( + api: &WorkspaceApi, + record: &TicketRoleAssignmentRecord, + operation_id: &str, + selected: &[server_api::TicketWorkerAttachmentBinding], + delegation: Option<&TicketStartDelegation>, ) -> ApiResult { let TicketAssignmentPrincipal::Worker { runtime_id, @@ -13997,7 +14602,13 @@ async fn scoped_set_ticket_assignment_with_context( request: Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let (author, source) = authenticated_assignment_source(&api, &headers, context)?; + let source_api = api.clone(); + let context = context.clone(); + let (author, source) = api + .host_executors + .host()? + .run(move || authenticated_assignment_source(&source_api, &headers, &context)) + .await?; apply_set_ticket_assignment(State(api), path, request, author, Some(source)).await } @@ -14008,46 +14619,83 @@ async fn apply_set_ticket_assignment( assigned_by: String, source: Option, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = require_ticket_assignment_value("operation_id", request.operation_id)?; - let expected_assignment_id = request - .expected_assignment_id - .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) - .transpose()?; - let principal = internal_ticket_assignment_principal(request.principal)?; - let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let record = TicketRoleAssignmentRecord { - workspace_id: workspace_id.clone(), - ticket_id: ticket.id.clone(), - assignment_id: new_id("tasg"), - role, - principal, - assigned_by, - assigned_at, - }; - let assignment = match role { + let prep_api = api.clone(); + let prep_workspace = workspace_id.clone(); + let (ticket, role, operation_id, expected_assignment_id, record, workdir_bindings) = api + .host_executors + .host()? + .run(move || { + let api = &prep_api; + let workspace_id = prep_workspace; + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = + require_ticket_assignment_value("operation_id", request.operation_id)?; + let expected_assignment_id = request + .expected_assignment_id + .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) + .transpose()?; + let principal = internal_ticket_assignment_principal(request.principal)?; + let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let record = TicketRoleAssignmentRecord { + workspace_id: workspace_id.clone(), + ticket_id: ticket.id.clone(), + assignment_id: new_id("tasg"), + role, + principal, + assigned_by, + assigned_at, + }; + Ok::<_, ApiError>(( + ticket, + role, + operation_id, + expected_assignment_id, + record, + request.workdir_bindings, + )) + }) + .await?; + let (assignment, notification) = match role { TicketAssignmentRole::Orchestrator => { - if let Some(source) = source.as_ref() { - api.store.set_current_ticket_orchestrator_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - source, - )? - } else { - // Only the cfg(test) direct-handler adapter lacks a Backend source. - api.store.set_current_ticket_role_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - )? - } + let store = api.store.clone(); + let notification_api = api.clone(); + let record = record.clone(); + let expected = expected_assignment_id.clone(); + let operation_id = operation_id.clone(); + api.store + .db_executor() + .run(move || { + let expected_assignment_id = expected; + let assignment = if let Some(source) = source.as_ref() { + store.set_current_ticket_orchestrator_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + source, + ) + } else { + // Only the cfg(test) direct-handler adapter lacks a Backend source. + store.set_current_ticket_role_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + ) + }?; + // Dispatch from the accepted SQL owner after commit, even + // when its waiter has gone. The response still observes the + // same best-effort notification completion as before offload. + Ok::<_, Error>(( + assignment, + enqueue_orchestrator_assignment_attention(¬ification_api), + )) + }) + .await? } TicketAssignmentRole::Worker => { if expected_assignment_id.is_some() { @@ -14061,14 +14709,17 @@ async fn apply_set_ticket_assignment( source.as_ref(), request.orchestrator_assignment_id.as_deref(), )?; - start_manual_ticket_worker_assignment( - &api, - &record, - &operation_id, - &request.workdir_bindings, - delegation.as_ref(), + ( + start_manual_ticket_worker_assignment( + &api, + &record, + &operation_id, + &workdir_bindings, + delegation.as_ref(), + ) + .await?, + None, ) - .await? } TicketAssignmentRole::Owner | TicketAssignmentRole::Contributor => { return Err(Error::TicketAssignmentConflict( @@ -14078,12 +14729,8 @@ async fn apply_set_ticket_assignment( .into()); } }; - if role == TicketAssignmentRole::Orchestrator - && let Some(orchestrator) = find_workspace_orchestrator(&api) - { - // The Assign row/event/receipt has committed. Delivery failure cannot - // roll back delegation; startup/restore/idle use this same durable scan. - dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + if let Some(notification) = notification { + let _ = notification.await; } Ok(Json(server_api::TicketRoleAssignmentMutationResponse { workspace_id, @@ -14109,7 +14756,13 @@ async fn scoped_clear_ticket_assignment_with_context( query: Query, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let (author, source) = authenticated_assignment_source(&api, &headers, context)?; + let source_api = api.clone(); + let context = context.clone(); + let (author, source) = api + .host_executors + .host()? + .run(move || authenticated_assignment_source(&source_api, &headers, &context)) + .await?; apply_clear_ticket_assignment(State(api), path, query, author, Some(source)).await } @@ -14120,65 +14773,70 @@ async fn apply_clear_ticket_assignment( author: String, source: Option, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = query - .operation_id - .map(|value| require_ticket_assignment_value("operation_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) - })?; - let assignment_id = query - .assignment_id - .map(|value| require_ticket_assignment_value("assignment_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) - })?; - let event_id = new_id("tasev"); - let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let reason = Some("role assignment removed from Ticket detail"); - let cleared = if role == TicketAssignmentRole::Orchestrator - && let Some(source) = source.as_ref() - { - api.store.clear_current_ticket_orchestrator_assignment( - &workspace_id, - &ticket.id, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - source, - )? - } else { - api.store.clear_current_ticket_role_assignment( - &workspace_id, - &ticket.id, - role, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - )? - }; - if !cleared { - return Err(Error::TicketAssignmentConflict(format!( - "assignment `{assignment_id}` is not current for role `{}`", - role.as_str() - )) - .into()); - } - Ok(Json(server_api::TicketRoleAssignmentMutationResponse { - workspace_id, - ticket_id: ticket.id, - assignment: None, - })) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = query + .operation_id + .map(|value| require_ticket_assignment_value("operation_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) + })?; + let assignment_id = query + .assignment_id + .map(|value| require_ticket_assignment_value("assignment_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) + })?; + let event_id = new_id("tasev"); + let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let reason = Some("role assignment removed from Ticket detail"); + let cleared = if role == TicketAssignmentRole::Orchestrator + && let Some(source) = source.as_ref() + { + api.store.clear_current_ticket_orchestrator_assignment( + &workspace_id, + &ticket.id, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + source, + )? + } else { + api.store.clear_current_ticket_role_assignment( + &workspace_id, + &ticket.id, + role, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + )? + }; + if !cleared { + return Err(Error::TicketAssignmentConflict(format!( + "assignment `{assignment_id}` is not current for role `{}`", + role.as_str() + )) + .into()); + } + Ok(Json(server_api::TicketRoleAssignmentMutationResponse { + workspace_id, + ticket_id: ticket.id, + assignment: None, + })) + }) + .await } async fn scoped_cancel_ticket_implementation( @@ -14187,60 +14845,65 @@ async fn scoped_cancel_ticket_implementation( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let ticket = api.authority.ticket(&path.id)?; - let operation_id = require_ticket_assignment_value("operation_id", request.operation_id)?; - let assignment_id = require_ticket_assignment_value("assignment_id", request.assignment_id)?; - let reason = require_ticket_assignment_value("reason", request.reason)?; - if reason.len() > 512 { - return Err(Error::InvalidInput( - "implementation cancellation reason must be at most 512 bytes".to_string(), - ) - .into()); - } + // Runtime cancel and session cleanup are mixed work. One accepted Host + // owner retains authority through the whole cancellation transaction/detail. + run_host_workflow(api.host_executors.host()?, async move { + let ticket = api.authority.ticket(&path.id)?; + let operation_id = require_ticket_assignment_value("operation_id", request.operation_id)?; + let assignment_id = require_ticket_assignment_value("assignment_id", request.assignment_id)?; + let reason = require_ticket_assignment_value("reason", request.reason)?; + if reason.len() > 512 { + return Err(Error::InvalidInput( + "implementation cancellation reason must be at most 512 bytes".to_string(), + ) + .into()); + } - let current = api.store.get_active_ticket_role_assignment( - &path.workspace_id, - &ticket.id, - TicketAssignmentRole::Worker, - )?; - if let Some(assignment) = current.filter(|value| value.assignment_id == assignment_id) - && let TicketAssignmentPrincipal::Worker { - runtime_id, - worker_id, - } = assignment.principal - { - if api - .store - .get_ticket_assignment_operation(&path.workspace_id, &operation_id)? - .is_some() + let current = api.store.get_active_ticket_role_assignment( + &path.workspace_id, + &ticket.id, + TicketAssignmentRole::Worker, + )?; + if let Some(assignment) = current.filter(|value| value.assignment_id == assignment_id) + && let TicketAssignmentPrincipal::Worker { + runtime_id, + worker_id, + } = assignment.principal { + if api + .store + .get_ticket_assignment_operation(&path.workspace_id, &operation_id)? + .is_some() + { + return Err(Error::TicketAssignmentConflict(format!( + "operation `{operation_id}` was already used for another Ticket assignment mutation" + )) + .into()); + } + let worker = RuntimeWorkerRef::new(runtime_id, worker_id); + cancel_ticket_worker(&api, &worker, &reason).await?; + } + + let cancelled = api.store.cancel_current_ticket_worker_assignment( + &path.workspace_id, + &ticket.id, + &assignment_id, + &new_id("tasev"), + &new_id("tev"), + &operation_id, + "workspace-web", + &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + &reason, + )?; + if !cancelled { return Err(Error::TicketAssignmentConflict(format!( - "operation `{operation_id}` was already used for another Ticket assignment mutation" + "assignment `{assignment_id}` is not the current Coder implementation" )) .into()); } - let worker = RuntimeWorkerRef::new(runtime_id, worker_id); - cancel_ticket_worker(&api, &worker, &reason).await?; - } - - let cancelled = api.store.cancel_current_ticket_worker_assignment( - &path.workspace_id, - &ticket.id, - &assignment_id, - &new_id("tasev"), - &new_id("tev"), - &operation_id, - "workspace-web", - &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), - &reason, - )?; - if !cancelled { - return Err(Error::TicketAssignmentConflict(format!( - "assignment `{assignment_id}` is not the current Coder implementation" - )) - .into()); - } - browser_ticket_detail(&api, &ticket.id) + browser_ticket_detail(&api, &ticket.id) + }) + .await } async fn cancel_ticket_worker( @@ -14697,49 +15360,54 @@ async fn scoped_edit_ticket_item( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let mut targets = request.target.map(|target| match target { - server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { - targets: targets - .into_iter() - .map(|target| ticket::TicketTarget { - repository_key: target.repository_key, - ref_selector: target.ref_selector, - access: target.access, - }) - .collect(), - }, - server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, - }); - if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { - validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; - } - browser_ticket_backend(&api)? - .edit_item( - TicketIdOrSlug::Id(path.id.clone()), - TicketItemEdit { - title: request.title, - body: request.body.map(MarkdownText::new), - body_replacement: match (request.old_string, request.new_string) { - (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { - old_string, - new_string, - replace_all: request.replace_all, - }), - (None, None) => None, - _ => { - return Err(settings_bad_request( - "invalid_ticket_edit_replacement", - "old_string and new_string must be provided together", - )); - } + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let mut targets = request.target.map(|target| match target { + server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { + targets: targets + .into_iter() + .map(|target| ticket::TicketTarget { + repository_key: target.repository_key, + ref_selector: target.ref_selector, + access: target.access, + }) + .collect(), }, - targets, - author: request.author, - }, - ) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) + server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, + }); + if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { + validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; + } + browser_ticket_backend(&api)? + .edit_item( + TicketIdOrSlug::Id(path.id.clone()), + TicketItemEdit { + title: request.title, + body: request.body.map(MarkdownText::new), + body_replacement: match (request.old_string, request.new_string) { + (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { + old_string, + new_string, + replace_all: request.replace_all, + }), + (None, None) => None, + _ => { + return Err(settings_bad_request( + "invalid_ticket_edit_replacement", + "old_string and new_string must be provided together", + )); + } + }, + targets, + author: request.author, + }, + ) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_append_ticket_event( @@ -14747,21 +15415,26 @@ async fn scoped_append_ticket_event( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = match request.role { - server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, - server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, - server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, - server_api::BrowserTicketThreadRole::ImplementationReport => { - TicketEventKind::ImplementationReport - } - }; - let mut event = NewTicketEvent::new(role, request.body); - event.author = request.author; - browser_ticket_backend(&api)? - .add_event(TicketIdOrSlug::Id(path.id.clone()), event) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let role = match request.role { + server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, + server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, + server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, + server_api::BrowserTicketThreadRole::ImplementationReport => { + TicketEventKind::ImplementationReport + } + }; + let mut event = NewTicketEvent::new(role, request.body); + event.author = request.author; + browser_ticket_backend(&api)? + .add_event(TicketIdOrSlug::Id(path.id.clone()), event) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_mark_ticket_ready_from_browser( @@ -14771,14 +15444,32 @@ async fn scoped_mark_ticket_ready_from_browser( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let _ = scoped_mark_ticket_ready( - State(api.clone()), - AxumPath((path.workspace_id.clone(), path.id.clone())), - headers, - Json(request), - ) - .await?; - browser_ticket_detail(&api, &path.id) + api.host_executors + .host()? + .run(move || { + let result = execute_ticket_rest_operation_on_host( + &api, + &path.workspace_id, + headers, + None, + TicketBackendOperation::MarkReady { + id: TicketIdOrSlug::Query(path.id.clone()), + request: ticket::TicketMarkReady { + operation_key: request.operation_key, + reason: request.reason, + author: None, + intake_summary: request.intake_summary, + }, + }, + )?; + // Preserve result validation before projecting the committed detail. + let _ = ticket_rest_result(result, |result| match result { + TicketBackendOperationResult::Ticket(ticket) => Some(ticket), + _ => None, + })?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_queue_ticket( @@ -14940,17 +15631,24 @@ fn schedule_ticket_item_check( } }; let api = api.clone(); - tokio::task::spawn_blocking(move || { - if let Err(error) = api.dispatch_backend_job_reservation(reservation) { - tracing::warn!( - job_id = %request.job_id, - ticket_input_digest = ?request.input_digest(), - %error, - "asynchronous Ticket item checker dispatch failed" - ); - api.drain_reserved_backend_jobs(); - } + let admission = api.host_executors.host().and_then(|executor| { + executor.dispatch(move || { + if let Err(error) = api.dispatch_backend_job_reservation(reservation) { + tracing::warn!( + job_id = %request.job_id, + ticket_input_digest = ?request.input_digest(), + %error, + "asynchronous Ticket item checker dispatch failed" + ); + if !matches!(error, Error::DbOperationOutcomeUnknown) { + api.drain_reserved_backend_jobs(); + } + } + }) }); + if let Err(error) = admission { + tracing::warn!(%error, "Ticket checker was not admitted; its reservation remains durable"); + } } async fn execute_ticket_rest_operation( @@ -14967,7 +15665,7 @@ async fn execute_ticket_rest_operation_with_context( workspace_id: &str, headers: HeaderMap, context: Option<&server_api::ServerRequestContext>, - mut operation: TicketBackendOperation, + operation: TicketBackendOperation, ) -> ApiResult { validate_workspace_scope(api, workspace_id)?; if let TicketBackendOperation::UpdateCase { ticket, request } = operation { @@ -14986,6 +15684,28 @@ async fn execute_ticket_rest_operation_with_context( .await?; return Ok(TicketBackendOperationResult::Ticket(updated)); } + let job_api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let context = context.cloned(); + api.host_executors + .host()? + .run(move || { + let api = &job_api; + let workspace_id = workspace_id.as_str(); + let context = context.as_ref(); + execute_ticket_rest_operation_on_host(api, workspace_id, headers, context, operation) + }) + .await +} + +// Already-owned callers must not re-admit to the occupied Host lane. +fn execute_ticket_rest_operation_on_host( + api: &WorkspaceApi, + workspace_id: &str, + headers: HeaderMap, + context: Option<&server_api::ServerRequestContext>, + mut operation: TicketBackendOperation, +) -> ApiResult { validate_ticket_operation_repository_keys(api, workspace_id, &operation)?; let mut backend = SqliteTicketBackend::open_verified( api.config.database_path.clone(), @@ -16145,6 +16865,8 @@ async fn scoped_list_merge_requests( AxumPath(workspace_id): AxumPath, Query(query): Query, ) -> ApiResult> { + api.host_executors.host()?.run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; require_workspace_access(&workspace_id, &api)?; let ticket_id = query @@ -16253,6 +16975,8 @@ async fn scoped_list_merge_requests( items, next_cursor: page.next_cursor, })) + + }).await } fn merge_ref_diagnostic(error: ApiError) -> MergeRequestRefDiagnostic { @@ -16358,159 +17082,169 @@ async fn scoped_show_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let store = merge_request_store(&api, &workspace_id)?; - let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let integrated_refs = integrated_merge_refs(&mr); - mr.thread = store.thread_page_by_id( - &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - let ticket_id = mr.ticket_ids.first().ok_or_else(|| { - Error::MergeRequest(merge_request::MergeRequestError::Conflict( - "Merge Request has no linked Ticket".into(), - )) - })?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; - let (source, target) = if let Some(refs) = integrated_refs { - refs - } else { - let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { - (Some(selector), Some(assignment)) => { - match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - } - } - (Some(_), None) => unknown_merge_ref( - "source_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the source ref", - ), - (None, _) => server_api::MergeRequestRefResponse { - status: "requires_repair".into(), - resolved_ref: None, - observed_at: Utc::now().to_rfc3339(), - diagnostic: None, - }, - }; - let target = match assignment.as_ref() { - Some(assignment) => match observe_published_merge_ref( - &api, + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let store = merge_request_store(&api, &workspace_id)?; + let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let integrated_refs = integrated_merge_refs(&mr); + mr.thread = store.thread_page_by_id( &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - }, - None => unknown_merge_ref( - "target_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the target ref", - ), - }; - (source, target) - }; - let linked_tickets = mr - .ticket_ids - .iter() - .map(|ticket_id| { - Ok(server_api::MergeRequestLinkedTicketResponse { - ticket_id: ticket_id.clone(), - key: api.store.resource_key( - &workspace_id, - WorkspaceResourceKind::Ticket, - ticket_id, - )?, - }) + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + let ticket_id = mr.ticket_ids.first().ok_or_else(|| { + Error::MergeRequest(merge_request::MergeRequestError::Conflict( + "Merge Request has no linked Ticket".into(), + )) + })?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; + let (source, target) = if let Some(refs) = integrated_refs { + refs + } else { + let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { + (Some(selector), Some(assignment)) => { + match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + } + } + (Some(_), None) => unknown_merge_ref( + "source_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the source ref", + ), + (None, _) => server_api::MergeRequestRefResponse { + status: "requires_repair".into(), + resolved_ref: None, + observed_at: Utc::now().to_rfc3339(), + diagnostic: None, + }, + }; + let target = match assignment.as_ref() { + Some(assignment) => match observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + }, + None => unknown_merge_ref( + "target_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the target ref", + ), + }; + (source, target) + }; + let linked_tickets = mr + .ticket_ids + .iter() + .map(|ticket_id| { + Ok(server_api::MergeRequestLinkedTicketResponse { + ticket_id: ticket_id.clone(), + key: api.store.resource_key( + &workspace_id, + WorkspaceResourceKind::Ticket, + ticket_id, + )?, + }) + }) + .collect::>>()?; + let merge_request = public_merge_request(&api, &workspace_id, mr)?; + Ok(Json(server_api::MergeRequestDetailResponse { + merge_request, + source, + target, + linked_tickets, + })) }) - .collect::>>()?; - let merge_request = public_merge_request(&api, &workspace_id, mr)?; - Ok(Json(server_api::MergeRequestDetailResponse { - merge_request, - source, - target, - linked_tickets, - })) + .await } async fn scoped_merge_request_readiness( State(api): State, AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; - let (current_subject_ref, source_blocker) = - if mr.state == merge_request::MergeRequestState::Merged { - ( - mr.merged_result() - .ok() - .map(|merge| merge.approved_source_ref.clone()), - None, - ) - } else { - match (mr.selector_from.as_deref(), assignment) { - (Some(selector), Some(assignment)) => match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => (Some(observation.resolved_ref), None), - Err(error) => { - let diagnostic = merge_ref_diagnostic(error); - let blocker = source_ref_readiness_blocker(&diagnostic.code); - (None, Some(blocker.to_string())) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; + let (current_subject_ref, source_blocker) = + if mr.state == merge_request::MergeRequestState::Merged { + ( + mr.merged_result() + .ok() + .map(|merge| merge.approved_source_ref.clone()), + None, + ) + } else { + match (mr.selector_from.as_deref(), assignment) { + (Some(selector), Some(assignment)) => match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => (Some(observation.resolved_ref), None), + Err(error) => { + let diagnostic = merge_ref_diagnostic(error); + let blocker = source_ref_readiness_blocker(&diagnostic.code); + (None, Some(blocker.to_string())) + } + }, + (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), + (None, _) => (None, None), } + }; + let mut report = store.readiness(merge_request::ReadinessCheck { + merge_request_id, + ticket_id, + current_subject_ref, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: String::new(), + worker_id: String::new(), + assignment_id: String::new(), }, - (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), - (None, _) => (None, None), + })?; + if let Some(blocker) = source_blocker { + report + .blockers + .retain(|current| current != "selector_unresolved"); + if !report.blockers.contains(&blocker) { + report.blockers.push(blocker); + } + report.ready = false; } - }; - let mut report = store.readiness(merge_request::ReadinessCheck { - merge_request_id, - ticket_id, - current_subject_ref, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: String::new(), - worker_id: String::new(), - assignment_id: String::new(), - }, - })?; - if let Some(blocker) = source_blocker { - report - .blockers - .retain(|current| current != "selector_unresolved"); - if !report.blockers.contains(&blocker) { - report.blockers.push(blocker); - } - report.ready = false; - } - Ok(Json(server_api::MergeRequestReadinessResponse { - ready: report.ready, - blockers: report.blockers, - subject_ref: report.subject_ref, - review: report.review.map(public_review_event), - })) + Ok(Json(server_api::MergeRequestReadinessResponse { + ready: report.ready, + blockers: report.blockers, + subject_ref: report.subject_ref, + review: report.review.map(public_review_event), + })) + }) + .await } async fn scoped_open_merge_request( @@ -16519,71 +17253,79 @@ async fn scoped_open_merge_request( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let repository = api.require_configured_workspace_repository_by_key(&input.repository_key)?; - let repository_id = repository.id; - require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_from, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &repository_id, - &input.selector_from, - &source_observation.resolved_ref, - )?; - observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_to, - )?; - let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( - merge_request::OpenMergeRequest { - merge_request_id: Uuid::now_v7().to_string(), - ticket_id, - repository_id: repository_id.clone(), - selector_from: input.selector_from, - selector_to: input.selector_to, - summary: input.summary, - auth: merge_request::MergeRequestAuth { - workspace_id: workspace_id.clone(), - repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }, - )?; - Ok(Json(public_merge_request( - &api, - &workspace_id, - merge_request, - )?)) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let repository = + api.require_configured_workspace_repository_by_key(&input.repository_key)?; + let repository_id = repository.id; + require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_from, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &repository_id, + &input.selector_from, + &source_observation.resolved_ref, + )?; + observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_to, + )?; + let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( + merge_request::OpenMergeRequest { + merge_request_id: Uuid::now_v7().to_string(), + ticket_id, + repository_id: repository_id.clone(), + selector_from: input.selector_from, + selector_to: input.selector_to, + summary: input.summary, + auth: merge_request::MergeRequestAuth { + workspace_id: workspace_id.clone(), + repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }, + )?; + Ok(Json(public_merge_request( + &api, + &workspace_id, + merge_request, + )?)) + }) + .await } async fn scoped_merge_request_thread( @@ -16591,19 +17333,24 @@ async fn scoped_merge_request_thread( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult>> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( - &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - Ok(Json( - events - .into_iter() - .map(public_merge_request_thread_event) - .collect(), - )) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( + &workspace_id, + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + Ok(Json( + events + .into_iter() + .map(public_merge_request_thread_event) + .collect(), + )) + }) + .await } async fn scoped_repair_merge_request_selector( @@ -16619,37 +17366,46 @@ async fn scoped_repair_merge_request_selector( if !input.explicit_confirmation { return Err(Error::BrowserReopenConfirmationRequired.into()); } - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - let resolved_subject_ref = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &input.selector_from, - )? - .resolved_ref; - let repaired = store.repair_selector_from(merge_request::RepairSelectorFrom { - workspace_id: workspace_id.clone(), - merge_request_id, - ticket_id, - selector_from: input.selector_from, - resolved_subject_ref, - repaired_by: merge_request::WorkerIdentity { - runtime_id: "browser".into(), - worker_id: "authenticated-user".into(), - }, - reason: input.reason, - now: Utc::now(), - })?; - Ok(Json(public_merge_request(&api, &workspace_id, repaired)?)) + // Provider observation and intact repair/audit SQL share an accepted owner; + // authentication's earlier DB await does not offload this mixed stage. + api.host_executors + .host()? + .run(move || { + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + let resolved_subject_ref = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &input.selector_from, + )? + .resolved_ref; + let repaired = store.repair_selector_from(merge_request::RepairSelectorFrom { + workspace_id: workspace_id.clone(), + merge_request_id, + ticket_id, + selector_from: input.selector_from, + resolved_subject_ref, + repaired_by: merge_request::WorkerIdentity { + runtime_id: "browser".into(), + worker_id: "authenticated-user".into(), + }, + reason: input.reason, + now: Utc::now(), + })?; + Ok(Json(public_merge_request(&api, &workspace_id, repaired)?)) + }) + .await } async fn scoped_register_reviewer_child_session( @@ -16658,20 +17414,25 @@ async fn scoped_register_reviewer_child_session( AxumPath(workspace_id): AxumPath, Json(input): Json, ) -> ApiResult { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( - merge_request::RegisterReviewerChildSession { - workspace_id, - parent_runtime_id: source.runtime_id, - parent_worker_id: source.worker_id, - child_session_id: input.child_session_id, - reviewer_profile: "builtin:reviewer".into(), - now: Utc::now(), - }, - )?; - Ok(StatusCode::NO_CONTENT) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( + merge_request::RegisterReviewerChildSession { + workspace_id, + parent_runtime_id: source.runtime_id, + parent_worker_id: source.worker_id, + child_session_id: input.child_session_id, + reviewer_profile: "builtin:reviewer".into(), + now: Utc::now(), + }, + )?; + Ok(StatusCode::NO_CONTENT) + }) + .await } async fn scoped_register_merge_request_review_capability( @@ -16680,107 +17441,115 @@ async fn scoped_register_merge_request_review_capability( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let requested_ticket_id = - resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; - if requested_ticket_id != ticket_id { - return Err(Error::TicketAssignmentConflict( - "Reviewer handoff Ticket does not match the addressed Merge Request".into(), - ) - .into()); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &mr.repository_id, - selector, - &source_observation.resolved_ref, - )?; - source_observation.resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err( - Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), - ); - } - }; - let mut ticket_merge_request_subjects = Vec::new(); - for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { - let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged { - merged_result_source_ref(&linked)? - } else { - let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { - Error::InvalidInput(format!( - "Merge Request `{}` selector_from requires repair", - linked.merge_request_id - )) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let requested_ticket_id = + resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; + if requested_ticket_id != ticket_id { + return Err(Error::TicketAssignmentConflict( + "Reviewer handoff Ticket does not match the addressed Merge Request".into(), + ) + .into()); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput("selector_from requires repair".into()) + })?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &mr.repository_id, + selector, + &source_observation.resolved_ref, + )?; + source_observation.resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err(Error::InvalidInput( + "closed Merge Request is not reviewable".into(), + ) + .into()); + } + }; + let mut ticket_merge_request_subjects = Vec::new(); + for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { + let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged + { + merged_result_source_ref(&linked)? + } else { + let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput(format!( + "Merge Request `{}` selector_from requires repair", + linked.merge_request_id + )) + })?; + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &linked.repository_id, + linked_selector, + )? + .resolved_ref + }; + ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { + merge_request_id: linked.merge_request_id, + subject_ref: linked_subject_ref, + }); + } + let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; + store.request_review(merge_request::RequestMergeRequestReview { + merge_request_id, + ticket_id, + ticket_content_digest, + ticket_merge_request_subjects, + subject_ref, + child_session_id: input.child_session_id, + capability_token: input.capability_token, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), })?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &linked.repository_id, - linked_selector, - )? - .resolved_ref - }; - ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { - merge_request_id: linked.merge_request_id, - subject_ref: linked_subject_ref, - }); - } - let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; - store.request_review(merge_request::RequestMergeRequestReview { - merge_request_id, - ticket_id, - ticket_content_digest, - ticket_merge_request_subjects, - subject_ref, - child_session_id: input.child_session_id, - capability_token: input.capability_token, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - })?; - Ok(StatusCode::NO_CONTENT) + Ok(StatusCode::NO_CONTENT) + }) + .await } async fn scoped_submit_merge_request_review( @@ -16788,67 +17557,74 @@ async fn scoped_submit_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let review_authorization = - store.authorize_review_submission(&merge_request_id, &input.capability_token)?; - if review_authorization.workspace_id != workspace_id - || review_authorization.merge_request_id != merge_request_id - { - return Err( - Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( - "review grant invalid".into(), - )) - .into(), - ); - } - let ticket_id = review_authorization.ticket_id; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - let current_subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err( - Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), - ); - } - }; - let decision = internal_review_decision(input.decision); - let findings = input - .findings - .into_iter() - .map(internal_review_finding) - .collect(); - let event = store.submit_review(merge_request::SubmitMergeRequestReview { - merge_request_id, - ticket_id, - current_subject_ref, - capability_token: input.capability_token, - decision, - body: input.body, - findings, - now: Utc::now(), - })?; - Ok(Json(public_review_event(event))) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let review_authorization = + store.authorize_review_submission(&merge_request_id, &input.capability_token)?; + if review_authorization.workspace_id != workspace_id + || review_authorization.merge_request_id != merge_request_id + { + return Err( + Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( + "review grant invalid".into(), + )) + .into(), + ); + } + let ticket_id = review_authorization.ticket_id; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + let current_subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput("selector_from requires repair".into()) + })?; + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err(Error::InvalidInput( + "closed Merge Request is not reviewable".into(), + ) + .into()); + } + }; + let decision = internal_review_decision(input.decision); + let findings = input + .findings + .into_iter() + .map(internal_review_finding) + .collect(); + let event = store.submit_review(merge_request::SubmitMergeRequestReview { + merge_request_id, + ticket_id, + current_subject_ref, + capability_token: input.capability_token, + decision, + body: input.body, + findings, + now: Utc::now(), + })?; + Ok(Json(public_review_event(event))) + }) + .await } async fn scoped_revoke_merge_request_review( @@ -16857,44 +17633,51 @@ async fn scoped_revoke_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - if !input.explicit_confirmation { - return Err(Error::BrowserReopenConfirmationRequired.into()); - } - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let event = store.revoke_review(merge_request::RevokeMergeRequestReview { - merge_request_id, - ticket_id, - review_event_id: input.review_event_id, - reason: input.reason, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - })?; - Ok(Json(public_review_revoked_event(event))) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + if !input.explicit_confirmation { + return Err(Error::BrowserReopenConfirmationRequired.into()); + } + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let event = store.revoke_review(merge_request::RevokeMergeRequestReview { + merge_request_id, + ticket_id, + review_event_id: input.review_event_id, + reason: input.reason, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + })?; + Ok(Json(public_review_revoked_event(event))) + }) + .await } async fn scoped_complete_merge_request( @@ -16903,94 +17686,101 @@ async fn scoped_complete_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - require_online_workspace_orchestrator_source(&api, &source)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { - let replay = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: existing.approved_source_ref.clone(), - target_ref_before: input.target_ref_before, - target_ref_after: input.target_ref_after, - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: String::new(), - }, - now: Utc::now(), - }; - return store - .complete(replay) - .map(public_merge_event) - .map(Json) - .map_err(Into::into); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - let current_source_ref = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref; - let observed_target = observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - )?; - require_completed_target_observation( - &observed_target.resolved_ref, - &input.target_ref_before, - &input.target_ref_after, - )?; - let completion = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: current_source_ref, - target_ref_before: input.target_ref_before.clone(), - target_ref_after: input.target_ref_after.clone(), - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }; - store.validate_completion(&completion)?; - store - .complete(completion) - .map(public_merge_event) - .map(Json) - .map_err(Into::into) + api.host_executors + .host()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + require_online_workspace_orchestrator_source(&api, &source)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { + let replay = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: existing.approved_source_ref.clone(), + target_ref_before: input.target_ref_before, + target_ref_after: input.target_ref_after, + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: String::new(), + }, + now: Utc::now(), + }; + return store + .complete(replay) + .map(public_merge_event) + .map(Json) + .map_err(Into::into); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + let selector = mr + .selector_from + .as_deref() + .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; + let current_source_ref = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref; + let observed_target = observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + )?; + require_completed_target_observation( + &observed_target.resolved_ref, + &input.target_ref_before, + &input.target_ref_after, + )?; + let completion = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: current_source_ref, + target_ref_before: input.target_ref_before.clone(), + target_ref_after: input.target_ref_after.clone(), + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }; + store.validate_completion(&completion)?; + store + .complete(completion) + .map(public_merge_event) + .map(Json) + .map_err(Into::into) + }) + .await } // Header-only entry point is retained solely for explicit legacy internal tests. @@ -17013,47 +17803,56 @@ async fn scoped_ticket_assignment_tracking_update_with_context( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(update): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let source = authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { - Error::WorkspacePermissionDenied( - "Tracking is self-reported by a verified Runtime Worker".into(), - ) - })?; - // Tracking authority is retained responsibility plus the registered singleton, - // NOT Ticket openness, active work, Worker state or a model-supplied role. - if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { - return Err(Error::WorkspacePermissionDenied( - "Tracking updates require the registered current Workspace Orchestrator".into(), - ) - .into()); - } - let authentication_binding_id = context - .runtime_source - .as_ref() - .map(|source| source.authentication_binding_id.clone()) - .ok_or_else(|| { - Error::WorkspacePermissionDenied( + let context = context.clone(); + api.host_executors + .host()? + .run(move || { + let context = &context; + + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let source = + authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Tracking is self-reported by a verified Runtime Worker".into(), + ) + })?; + // Tracking authority is retained responsibility plus the registered singleton, + // NOT Ticket openness, active work, Worker state or a model-supplied role. + if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { + return Err(Error::WorkspacePermissionDenied( + "Tracking updates require the registered current Workspace Orchestrator".into(), + ) + .into()); + } + let authentication_binding_id = context + .runtime_source + .as_ref() + .map(|source| source.authentication_binding_id.clone()) + .ok_or_else(|| { + Error::WorkspacePermissionDenied( "Tracking requires the Runtime binding actually used to authenticate the request" .into(), ) - })?; - let source_context = ticket::TicketAssignmentTrackingSourceContext { - actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - agent_key: "workspace-orchestrator".into(), - authentication_binding_id, - }; - // The Ticket-owned mutation checks current assignment/CAS and exact replay in - // one transaction. A historical receipt never restores operational grants. - let value = api.store.update_ticket_assignment_tracking( - &workspace_id, - &ticket_id, - &update, - &source_context, - )?; - Ok(Json(value)) + })?; + let source_context = ticket::TicketAssignmentTrackingSourceContext { + actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + agent_key: "workspace-orchestrator".into(), + authentication_binding_id, + }; + // The Ticket-owned mutation checks current assignment/CAS and exact replay in + // one transaction. A historical receipt never restores operational grants. + let value = api.store.update_ticket_assignment_tracking( + &workspace_id, + &ticket_id, + &update, + &source_context, + )?; + Ok(Json(value)) + }) + .await } async fn scoped_ticket_assignment_tracking_history( @@ -17061,25 +17860,31 @@ async fn scoped_ticket_assignment_tracking_history( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let limit = query.limit.unwrap_or(50); - if !(1..=100).contains(&limit) { - return Err( - Error::InvalidInput("tracking history limit must be between 1 and 100".into()).into(), - ); - } - let page = api.store.page_ticket_assignment_tracking_history( - &workspace_id, - &ticket_id, - &query.assignment_id, - query.after_event_id.as_deref(), - limit, - )?; - Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { - items: page.items, - next_event_id: page.next_event_id, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let limit = query.limit.unwrap_or(50); + if !(1..=100).contains(&limit) { + return Err(Error::InvalidInput( + "tracking history limit must be between 1 and 100".into(), + ) + .into()); + } + let page = api.store.page_ticket_assignment_tracking_history( + &workspace_id, + &ticket_id, + &query.assignment_id, + query.after_event_id.as_deref(), + limit, + )?; + Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { + items: page.items, + next_event_id: page.next_event_id, + })) + }) + .await } async fn scoped_ticket_case_update_with_context( @@ -17148,6 +17953,10 @@ async fn apply_ticket_case_update( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(mut request): Json, ) -> ApiResult> { + let actor = actor.cloned(); + api.host_executors.host()?.run(move || { + let actor = actor.as_ref(); + validate_workspace_scope(&api, &workspace_id)?; let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; let mut backend = browser_ticket_backend(&api)?; @@ -17204,6 +18013,8 @@ async fn apply_ticket_case_update( .update_case(&ticket_id, request) .map_err(Error::from)?; Ok(Json(ticket)) + + }).await } fn reject_non_browser_reopen_auth(headers: &HeaderMap) -> Result<()> { @@ -17744,6 +18555,20 @@ fn authenticate_worker_mutation_source( Ok(worker) } +async fn authenticate_worker_mutation_source_owned( + api: &WorkspaceApi, + workspace_id: &str, + headers: &HeaderMap, +) -> Result { + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let headers = headers.clone(); + api.host_executors + .host()? + .run(move || authenticate_worker_mutation_source(&api, &workspace_id, &headers)) + .await +} + fn current_worker_identity( api: &WorkspaceApi, workspace_id: &str, @@ -17771,44 +18596,93 @@ fn runtime_local_owner_worker_id<'a>( (caller.runtime_id == target_runtime_id).then_some(caller.worker_id.as_str()) } +// Locked async workflows stay on Tokio. Each accepted synchronous mutation +// retains this same owned guard through execution and discarded-result cleanup. +async fn run_worker_workdir_db_stage_locked( + api: &WorkspaceApi, + session_guard: &Arc>, + operation: impl FnOnce(WorkspaceApi) -> std::result::Result + Send + 'static, +) -> std::result::Result +where + T: Send + 'static, + E: From + Send + 'static, +{ + let stage_api = api.clone(); + let stage_guard = session_guard.clone(); + let (outcome, _session_guard) = api + .store + .db_executor() + .run(move || { + let outcome = operation(stage_api); + // Keep the guard with the outcome, including errors. If the waiter was + // cancelled, the job drops the outcome before releasing this guard. + Ok::<_, E>((outcome, stage_guard)) + }) + .await?; + outcome +} + async fn open_current_worker_workdir_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, ) -> Result { - let workdir = api + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_link = link.clone(); + let (workdir, effective_capabilities) = api .store - .list_workdir_registry(&api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == link.workdir_id) - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!( - "attached Workdir {} is not registered in this Workspace", - link.workdir_id - ), - })?; - if matches!( - workdir.source, - WorkdirRegistrySource::WorkspaceConfig { .. } - ) { - return Err(Error::WorkspacePermissionDenied( - "Logical config has no filesystem session".into(), - )); - } - let effective_capabilities = - effective_worker_workdir_capabilities(api, worker, &workdir, link.capabilities)?; + .db_executor() + .run(move || { + let workdir = stage_api + .store + .list_workdir_registry(&stage_api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == stage_link.workdir_id) + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: stage_worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!( + "attached Workdir {} is not registered in this Workspace", + stage_link.workdir_id + ), + })?; + if matches!( + workdir.source, + WorkdirRegistrySource::WorkspaceConfig { .. } + ) { + return Err(Error::WorkspacePermissionDenied( + "Logical config has no filesystem session".into(), + )); + } + let capabilities = effective_worker_workdir_capabilities( + &stage_api, + &stage_worker, + &workdir, + stage_link.capabilities, + )?; + Ok((workdir, capabilities)) + }) + .await?; if let WorkdirRegistrySource::ExternalGrant { grant_id } = &workdir.source { close_current_worker_attachment_session_locked(api, worker).await?; + let stage_api = api.clone(); + let stage_grant_id = grant_id.clone(); let mut grant = api .store - .get_external_workdir_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "external_workdir_grant_not_found".to_string(), - message: "External Workdir grant is unavailable".to_string(), - })?; + .db_executor() + .run(move || { + stage_api + .store + .get_external_workdir_grant(&stage_api.config.workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "external_workdir_grant_not_found".to_string(), + message: "External Workdir grant is unavailable".to_string(), + }) + }) + .await?; let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(api, grant_id) @@ -17816,9 +18690,13 @@ async fn open_current_worker_workdir_session_locked( .map_err(|error| error.error)?; } if matches!(grant.status.as_str(), "revoked" | "expired") { - cleanup_external_workdir_attachments_with_lock(api, &grant.workdir_id, Some(worker)) - .await - .map_err(|error| error.error)?; + cleanup_external_workdir_attachments_with_lock( + api, + &grant.workdir_id, + Some((worker, session_guard)), + ) + .await + .map_err(|error| error.error)?; cancel_external_workdir_expiry(api, grant_id); } if grant.status != "online" { @@ -17885,31 +18763,63 @@ async fn open_current_worker_workdir_session_locked( }); } }; + let stage_api = api.clone(); + let stage_repository_id = repository_id.to_string(); let repository = api - .require_configured_workspace_repository(repository_id) - .map_err(|error| error.error)?; + .store + .db_executor() + .run(move || { + stage_api + .require_configured_workspace_repository(&stage_repository_id) + .map_err(|error| error.error) + }) + .await?; let repository_requires_access = repository.source.kind == server_api::RepositorySourceKind::Ssh; if repository_requires_access { close_current_worker_attachment_session_locked(api, worker).await?; - let access = repository_access_request_for_workdir( - api, - runtime_id, - &workdir.workdir_id, - &format!( - "workdir-session:{}:{}:{}", - worker.runtime_id, worker.worker_id, workdir.workdir_id - ), - ) - .map_err(|error| error.error)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: runtime_id.to_string(), - code: "working_directory_remote_repository_access_required".to_string(), - message: "current Repository SSH access authority is unavailable".to_string(), - })?; - api.runtime - .authorize_working_directory_repository_access(runtime_id, access) - .map_err(RuntimeRegistryError::into_error)?; + let stage_api = api.clone(); + let stage_runtime_id = runtime_id.to_string(); + let stage_workdir_id = workdir.workdir_id.clone(); + let operation_id = format!( + "workdir-session:{}:{}:{}", + worker.runtime_id, worker.worker_id, workdir.workdir_id + ); + let stage_guard = session_guard.clone(); + let access = api + .store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + repository_access_request_for_workdir( + &stage_api, + &stage_runtime_id, + &stage_workdir_id, + &operation_id, + ) + .map_err(|error| error.error)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: stage_runtime_id, + code: "working_directory_remote_repository_access_required".to_string(), + message: "current Repository SSH access authority is unavailable".to_string(), + }) + }) + .await?; + let stage_api = api.clone(); + let stage_runtime_id = runtime_id.to_string(); + // The accepted Runtime authorization must retain the actual session lock, + // not just the async waiter that submitted it. + let stage_guard = session_guard.clone(); + api.host_executors + .host()? + .run(move || { + let _session_guard = stage_guard; + stage_api + .runtime + .authorize_working_directory_repository_access(&stage_runtime_id, access) + .map_err(RuntimeRegistryError::into_error) + }) + .await?; } else { // The registry is an operation-session cache, not routing authority. // Close any prior alias session before opening the explicitly selected @@ -18038,10 +18948,10 @@ fn restore_logical_workdir_attachments( .map_err(ApiError::from) } -fn sync_runtime_worker_workdir_attachments( +fn prepare_runtime_worker_workdir_attachments( api: &WorkspaceApi, worker: &RuntimeWorkerRef, -) -> ApiResult<()> { +) -> ApiResult> { let attachments = api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? @@ -18084,6 +18994,14 @@ fn sync_runtime_worker_workdir_attachments( }) }) .collect::>>()?; + Ok(attachments) +} + +fn replace_runtime_worker_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + attachments: Vec, +) -> ApiResult<()> { let result = api .runtime .replace_worker_workdir_attachments(worker, attachments) @@ -18103,12 +19021,68 @@ fn sync_runtime_worker_workdir_attachments( Ok(()) } +#[cfg(test)] +fn sync_runtime_worker_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ApiResult<()> { + let attachments = prepare_runtime_worker_workdir_attachments(api, worker)?; + replace_runtime_worker_workdir_attachments(api, worker, attachments) +} + +async fn sync_runtime_worker_workdir_attachments_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + session_guard: &Arc>, +) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.clone(); + let attachments = api + .store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + prepare_runtime_worker_workdir_attachments(&stage_api, &stage_worker) + }) + .await?; + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.clone(); + api.host_executors + .host()? + .run(move || { + let _session_guard = stage_guard; + replace_runtime_worker_workdir_attachments(&stage_api, &stage_worker, attachments) + }) + .await +} + async fn refresh_current_worker_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, + session_guard: &Arc>, ) -> Result<()> { close_current_worker_attachment_session_locked(api, worker).await?; + let stage_api = api.clone(); + let stage_worker = worker.clone(); let remaining = api + .store + .db_executor() + .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) + .await?; + if remaining.len() == 1 { + open_current_worker_workdir_session_locked(api, worker, &remaining[0], session_guard) + .await?; + } + Ok(()) +} + +fn active_filesystem_workdir_links( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Result> { + Ok(api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() @@ -18122,11 +19096,7 @@ async fn refresh_current_worker_session_locked( !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) }) }) - .collect::>(); - if remaining.len() == 1 { - open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await?; - } - Ok(()) + .collect()) } #[derive(Debug, Deserialize)] @@ -18141,35 +19111,107 @@ async fn scoped_submit_backend_job_result( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; - let binding = api - .store - .worker_registry_projection(&api.config.workspace_id, &worker)? - .and_then(|projection| projection.job) - .ok_or_else(|| { - Error::InvalidInput( - "current Worker is not bound to an active Backend Job attempt".to_string(), - ) - })?; - let attempt = api - .store - .get_backend_job_attempt( + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let binding = api + .store + .worker_registry_projection(&api.config.workspace_id, &worker)? + .and_then(|projection| projection.job) + .ok_or_else(|| { + Error::InvalidInput( + "current Worker is not bound to an active Backend Job attempt".to_string(), + ) + })?; + let attempt = api + .store + .get_backend_job_attempt( + &api.config.workspace_id, + &binding.job_id, + &binding.attempt_id, + )? + .ok_or_else(|| { + Error::Store("Backend Job attempt binding disappeared".to_string()) + })?; + let acceptance = api.accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + input_digest: attempt.input_digest, + result: request.result, + }, + )?; + Ok(Json(acceptance)) + }) + .await +} + +async fn detach_worker_workdir_stage_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + workdir_id: &str, + session_guard: &Arc>, +) -> Result> { + let worker = worker.clone(); + let workdir_id = workdir_id.to_string(); + run_worker_workdir_db_stage_locked(api, session_guard, move |api| { + api.store.detach_worker_workdir( &api.config.workspace_id, - &binding.job_id, - &binding.attempt_id, - )? - .ok_or_else(|| Error::Store("Backend Job attempt binding disappeared".to_string()))?; - let acceptance = api.accept_backend_job_result( - &worker, - &BackendJobResultSubmission { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - input_digest: attempt.input_digest, - result: request.result, - }, - )?; - Ok(Json(acceptance)) + &worker, + Some(&workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) + }) + .await +} + +async fn import_current_worker_workdir_observation( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + session_guard: Option<&Arc>>, +) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let has_history = api + .store + .db_executor() + .run(move || { + stage_api + .store + .worker_workdir_link_history_exists(&stage_api.config.workspace_id, &stage_worker) + }) + .await?; + if !has_history { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.cloned(); + let observed_worker = api + .host_executors + .host()? + .run(move || { + let _session_guard = stage_guard; + stage_api + .runtime + .worker(&stage_worker) + .map_err(RuntimeRegistryError::into_error) + }) + .await?; + if !observed_worker.workdir_attachments.is_empty() { + let stage_api = api.clone(); + let stage_guard = session_guard.cloned(); + api.store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + sync_worker_observation(&stage_api, &observed_worker) + }) + .await?; + } + } + Ok(()) } async fn scoped_attach_current_worker_workdir( @@ -18178,115 +19220,141 @@ async fn scoped_attach_current_worker_workdir( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host()? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &path.workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &path.workspace_id, + &headers, + )?) + }) + .await?; let alias = workdir::WorkdirAttachmentAlias::new(request.alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; let workdir_id = request.working_directory_id.trim(); if workdir_id.is_empty() || workdir_id.chars().any(char::is_control) { return Err(Error::InvalidRecordId(request.working_directory_id).into()); } - let workdir = api + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_workdir_id = workdir_id.to_string(); + let capabilities = api .store - .list_workdir_registry(&api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == workdir_id) - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!("unknown Workdir `{workdir_id}`"), - })?; - // A failed removal must not prevent its owner from attaching a still usable - // repository checkout to resolve current blockers. Runtime binding validates - // the actual checkout; durable pending-removal reservations still fence this - // mutation. Other sources and missing/corrupted materializations stay denied. - let repository_retry = workdir.materialization_status == "pending" - && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); - if workdir.materialization_status != "present" && !repository_retry { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir `{workdir_id}` is not available for attachment ({})", - workdir.materialization_status - )) - .into()); - } - match &workdir.source { - WorkdirRegistrySource::Repository { .. } => {} - WorkdirRegistrySource::WorkspaceConfig { .. } => { - return Err(Error::WorkspacePermissionDenied( - "Use the grant-bound Workspace config attachment operation".into(), - ) - .into()); - } - WorkdirRegistrySource::ExternalGrant { grant_id } => { - let grant = api + .db_executor() + .run(move || -> ApiResult<_> { + let api = stage_api; + let worker = stage_worker; + let workdir_id = stage_workdir_id.as_str(); + let workdir = api .store - .get_external_workdir_grant(&api.config.workspace_id, grant_id)? + .list_workdir_registry(&api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == workdir_id) .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir grant is unavailable".to_string(), + runtime_id: worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!("unknown Workdir `{workdir_id}`"), })?; - let grant_expired = external_workdir_grant_is_expired(&grant)?; - let online = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connection(grant_id) - .is_some_and(|connection| { - connection.generation == grant.generation - && connection.provider_instance_id == grant.provider_instance_id - && connection.workdir_id == workdir.workdir_id - && connection - .expires_at - .is_none_or(|expires_at| expires_at > Utc::now()) - }); - if grant.status != "online" || grant_expired || !online { - return Err(Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir provider is offline or expired".to_string(), - } + // A failed removal must not prevent its owner from attaching a still usable + // repository checkout to resolve current blockers. Runtime binding validates + // the actual checkout; durable pending-removal reservations still fence this + // mutation. Other sources and missing/corrupted materializations stay denied. + let repository_retry = workdir.materialization_status == "pending" + && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); + if workdir.materialization_status != "present" && !repository_retry { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir `{workdir_id}` is not available for attachment ({})", + workdir.materialization_status + )) .into()); } - } - } - let capabilities = ticket_target_capabilities_for_workdir(&api, &worker, &workdir)?; + match &workdir.source { + WorkdirRegistrySource::Repository { .. } => {} + WorkdirRegistrySource::WorkspaceConfig { .. } => { + return Err(Error::WorkspacePermissionDenied( + "Use the grant-bound Workspace config attachment operation".into(), + ) + .into()); + } + WorkdirRegistrySource::ExternalGrant { grant_id } => { + let grant = api + .store + .get_external_workdir_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir grant is unavailable".to_string(), + })?; + let grant_expired = external_workdir_grant_is_expired(&grant)?; + let online = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connection(grant_id) + .is_some_and(|connection| { + connection.generation == grant.generation + && connection.provider_instance_id == grant.provider_instance_id + && connection.workdir_id == workdir.workdir_id + && connection + .expires_at + .is_none_or(|expires_at| expires_at > Utc::now()) + }); + if grant.status != "online" || grant_expired || !online { + return Err(Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir provider is offline or expired".to_string(), + } + .into()); + } + } + } + Ok(ticket_target_capabilities_for_workdir( + &api, &worker, &workdir, + )?) + }) + .await?; let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; - if !api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, &worker)? - { - let observed_worker = api - .runtime - .worker(&worker) - .map_err(|error| error.into_error())?; - if !observed_worker.workdir_attachments.is_empty() { - sync_worker_observation(&api, &observed_worker)?; - } - } - if let Some(existing) = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == alias.as_str()) - { - if existing.workdir_id != workdir_id { - return Err(Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} attachment alias `{alias}` is already bound to Workdir {}", - worker.runtime_id, worker.worker_id, existing.workdir_id - )) - .into()); - } - let effective_capabilities = existing.capabilities.intersection(capabilities); - if effective_capabilities != existing.capabilities { - let mut attenuated = existing.clone(); - attenuated.capabilities = effective_capabilities; - api.store.attach_worker_workdir(&attenuated)?; - } - sync_runtime_worker_workdir_attachments(&api, &worker)?; + let _session_guard = Arc::new(session_lock.lock_owned().await); + import_current_worker_workdir_observation(&api, &worker, Some(&_session_guard)).await?; + let stage_worker = worker.clone(); + let stage_alias = alias.to_string(); + let stage_workdir_id = workdir_id.to_string(); + let existing = + run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| -> ApiResult<_> { + let existing = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias); + if let Some(existing) = existing { + if existing.workdir_id != stage_workdir_id { + return Err(Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} attachment alias `{stage_alias}` is already bound to Workdir {}", + stage_worker.runtime_id, stage_worker.worker_id, existing.workdir_id + )).into()); + } + let effective_capabilities = existing.capabilities.intersection(capabilities); + if effective_capabilities != existing.capabilities { + let mut attenuated = existing; + attenuated.capabilities = effective_capabilities; + api.store.attach_worker_workdir(&attenuated)?; + } + Ok(Some(effective_capabilities)) + } else { + Ok(None) + } + }) + .await?; + if let Some(effective_capabilities) = existing { + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; api.worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from)?; return Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { workspace_id: api.config.workspace_id.clone(), @@ -18297,7 +19365,7 @@ async fn scoped_attach_current_worker_workdir( })); } close_current_worker_attachment_session_locked(&api, &worker).await?; - let link = api.store.attach_worker_workdir(&WorkerWorkdirLinkRecord { + let new_link = WorkerWorkdirLinkRecord { connection_id: String::new(), workspace_id: api.config.workspace_id.clone(), worker: worker.clone(), @@ -18306,56 +19374,46 @@ async fn scoped_attach_current_worker_workdir( capabilities, linked_at: Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), unlinked_at: None, - })?; - if let Err(error) = sync_runtime_worker_workdir_attachments(&api, &worker) { - let _ = api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ); + }; + let link = run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| { + api.store.attach_worker_workdir(&new_link) + }) + .await?; + if let Err(error) = + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await + { + let _ = + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; return Err(error); } + let stage_api = api.clone(); + let stage_worker = worker.clone(); let active_count = api .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) - .filter(|link| { - api.store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) - .ok() - .flatten() - .is_none_or(|record| { - !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) - }) + .db_executor() + .run(move || { + active_filesystem_workdir_links(&stage_api, &stage_worker).map(|links| links.len()) }) - .count(); + .await?; if active_count == 1 - && let Err(error) = open_current_worker_workdir_session_locked(&api, &worker, &link).await + && let Err(error) = + open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await { - let _ = api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ); - let _ = sync_runtime_worker_workdir_attachments(&api, &worker); + let _ = + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; + let _ = + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await; return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from) { - api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - )?; - sync_runtime_worker_workdir_attachments(&api, &worker)?; - refresh_current_worker_session_locked(&api, &worker).await?; + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await?; + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; + refresh_current_worker_session_locked(&api, &worker, &_session_guard).await?; return Err(error); } Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -18485,7 +19543,12 @@ async fn scoped_detach_current_worker_workdir( validate_workspace_scope(&api, &workspace_id)?; let alias = workdir::WorkdirAttachmentAlias::new(alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; - let worker = current_worker_identity(&api, &workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = api + .host_executors + .host()? + .run(move || current_worker_identity(&stage_api, &workspace_id, &headers)) + .await?; detach_current_worker_workdir( &api, &worker, @@ -18496,6 +19559,18 @@ async fn scoped_detach_current_worker_workdir( .map(Json) } +async fn restore_worker_workdir_connection_stage_locked( + api: &WorkspaceApi, + link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, +) -> Result<()> { + let link = link.clone(); + run_worker_workdir_db_stage_locked(api, session_guard, move |api| { + api.store.restore_worker_workdir_connection(&link) + }) + .await +} + async fn detach_current_worker_workdir( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -18503,70 +19578,84 @@ async fn detach_current_worker_workdir( expected_connection_id: Option<&str>, ) -> ApiResult { let session_lock = current_worker_session_lock(api, worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_alias = alias.to_string(); + let expected_connection_id = expected_connection_id.map(str::to_string); let link = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == alias) - .ok_or_else(|| { - Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} has no Workdir attachment `{alias}`", - worker.runtime_id, worker.worker_id - )) - })?; - if expected_connection_id.is_some_and(|expected| expected != link.connection_id) { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir attachment `{alias}` connection changed during detach" - )) - .into()); - } + .db_executor() + .run(move || -> ApiResult<_> { + let link = stage_api + .store + .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias) + .ok_or_else(|| { + Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} has no Workdir attachment `{stage_alias}`", + stage_worker.runtime_id, stage_worker.worker_id + )) + })?; + if expected_connection_id + .as_ref() + .is_some_and(|expected| *expected != link.connection_id) + { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir attachment `{stage_alias}` connection changed during detach" + )) + .into()); + } + Ok(link) + }) + .await?; close_current_worker_attachment_session_locked(api, worker).await?; close_current_worker_command_sessions_for_alias_locked(api, worker, alias).await?; // Even unconditional HTTP requests condition the mutation on the lifetime just read // under the session lock, so a concurrent ledger replacement cannot be removed. - api.store.detach_worker_workdir_connection( - &api.config.workspace_id, - worker, - alias, - &link.connection_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - )?; - if let Err(error) = sync_runtime_worker_workdir_attachments(api, worker) { - api.store.restore_worker_workdir_connection(&link)?; + let stage_link = link.clone(); + run_worker_workdir_db_stage_locked(api, &_session_guard, move |api| { + api.store.detach_worker_workdir_connection( + &api.config.workspace_id, + &stage_link.worker, + &stage_link.alias, + &stage_link.connection_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) + }) + .await?; + if let Err(error) = + sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await + { + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; return Err(error); } + let stage_api = api.clone(); + let stage_worker = worker.clone(); let remaining = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) - .filter(|link| { - api.store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) - .ok() - .flatten() - .is_none_or(|record| { - !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) - }) - }) - .collect::>(); + .db_executor() + .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) + .await?; if remaining.len() == 1 && let Err(error) = - open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await + open_current_worker_workdir_session_locked(api, worker, &remaining[0], &_session_guard) + .await { - api.store.restore_worker_workdir_connection(&link)?; - let _ = sync_runtime_worker_workdir_attachments(api, worker); + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; + let _ = sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await; return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from) { - api.store.restore_worker_workdir_connection(&link)?; - sync_runtime_worker_workdir_attachments(api, worker)?; - refresh_current_worker_session_locked(api, worker).await?; + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; + sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await?; + refresh_current_worker_session_locked(api, worker, &_session_guard).await?; return Err(error); } Ok(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -18578,41 +19667,36 @@ async fn detach_current_worker_workdir( }) } -fn validated_current_worker_attachment( +async fn validated_current_worker_attachment_async( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, + session_guard: Option<&Arc>>, ) -> ApiResult { - let find = || -> Result> { - Ok(api + for imported in [false, true] { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_alias = target_workdir.to_string(); + let link = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == target_workdir)) - }; - if let Some(link) = find()? { - return Ok(link); - } - - // A Runtime can start the Worker immediately after reserving its local - // bindings, before the outer spawn handler projects them into the Backend. - // Import that exact alias set once; never substitute another attachment. - if !api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, worker)? - { - let observed_worker = api - .runtime - .worker(worker) - .map_err(|error| error.into_error())?; - if !observed_worker.workdir_attachments.is_empty() { - sync_worker_observation(api, &observed_worker)?; - if let Some(link) = find()? { - return Ok(link); - } + .db_executor() + .run(move || -> Result<_> { + Ok(stage_api + .store + .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias)) + }) + .await?; + if let Some(link) = link { + return Ok(link); + } + if !imported { + // A Runtime may start the Worker before the outer spawn handler + // projects its reserved aliases. Import that exact set only once. + import_current_worker_workdir_observation(api, worker, session_guard).await?; } } - Err(Error::WorkdirAttachmentConflict(format!( "Worker {}:{} has no Workdir attachment alias `{target_workdir}`", worker.runtime_id, worker.worker_id @@ -18721,19 +19805,24 @@ async fn scoped_execute_current_worker_workdir_operation( context.stage = "workspace_scope"; validate_workspace_scope(&api, &path.workspace_id)?; context.stage = "worker_identity"; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = api.host_executors.host()?.run(move || { + current_worker_identity(&stage_api, &path.workspace_id, &headers) + }).await?; context.runtime_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.runtime_id)); context.worker_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.worker_id)); let target_workdir = request.target_workdir; let result = match request.operation { WorkdirSessionOperation::CommandStart(command) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; + let link = validated_current_worker_attachment_async( + &api, &worker, &target_workdir, Some(&_session_guard), + ).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; context.stage = "provider_dispatch"; let provider_handle = source .start_command(command) @@ -18765,7 +19854,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandStatus(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; session .command_status(provider_handle) .await @@ -18774,7 +19863,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandOutput(mut output) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context).await?; output.handle = provider_handle; session .command_output(output) @@ -18784,7 +19873,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandCancel(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; session .cancel_command(provider_handle) .await @@ -18805,12 +19894,14 @@ async fn scoped_execute_current_worker_workdir_operation( | WorkdirSessionOperation::Glob(_) | WorkdirSessionOperation::Grep(_)) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; + let link = validated_current_worker_attachment_async( + &api, &worker, &target_workdir, Some(&_session_guard), + ).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; context.stage = "provider_dispatch"; execute_workdir_session_operation(&source, operation) .await @@ -18825,7 +19916,7 @@ async fn scoped_execute_current_worker_workdir_operation( }) } -fn current_worker_command_session( +async fn current_worker_command_session( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, @@ -18833,7 +19924,7 @@ fn current_worker_command_session( context: &mut server_api::WorkdirOperationLogContext, ) -> std::result::Result<(WorkdirSessionHandle, CommandHandle), InternalWorkdirOperationError> { context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(api, worker, target_workdir)?; + let link = validated_current_worker_attachment_async(api, worker, target_workdir, None).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "command_lookup"; let command = api @@ -18939,8 +20030,29 @@ async fn run_orchestrator_turn_end_hook(api: WorkspaceApi) { }; let mut worker_states = HashMap::new(); while let Some(update) = subscription.recv().await { - if !observe_orchestrator_turn_end(&api, &mut worker_states, update) { - return; + let stage_api = api.clone(); + let result = api.host_executors.host().map(|executor| async move { + executor + .run(move || -> Result<_> { + let keep_running = + observe_orchestrator_turn_end(&stage_api, &mut worker_states, update); + Ok((keep_running, worker_states)) + }) + .await + }); + match result { + Ok(operation) => match operation.await { + Ok((true, states)) => worker_states = states, + Ok((false, _)) => return, + Err(error) => { + tracing::warn!(%error, "Orchestrator hook stopped; durable inputs remain recoverable"); + return; + } + }, + Err(error) => { + tracing::warn!(%error, "Orchestrator hook executor unavailable"); + return; + } } } } @@ -19020,6 +20132,27 @@ fn maybe_dispatch_orchestrator_turn_end( } } +fn enqueue_orchestrator_assignment_attention( + api: &WorkspaceApi, +) -> Option> { + let api = api.clone(); + let (done, completion) = tokio::sync::oneshot::channel(); + match api.host_executors.host().and_then(|executor| { + executor.dispatch(move || { + if let Some(orchestrator) = find_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + let _ = done.send(()); + }) + }) { + Ok(()) => Some(completion), + Err(error) => { + tracing::warn!(%error, "Assign notification was not admitted; input remains durable"); + None + } + } +} + fn dispatch_orchestrator_assignment_attention(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { with_orchestrator_attention_reservation(api, worker, true, |state, pending| { check_orchestrator_assignment_attention(api, worker, state, pending); @@ -19360,15 +20493,20 @@ async fn scoped_get_memory_document( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let document = api.authority.memory_document()?; - Ok(Json(MemoryDocumentResponse { - bytes: document.body_md.len(), - body_md: document.body_md, - created_at: document.created_at, - updated_at: document.updated_at, - record_source: document.record_source, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let document = api.authority.memory_document()?; + Ok(Json(MemoryDocumentResponse { + bytes: document.body_md.len(), + body_md: document.body_md, + created_at: document.created_at, + updated_at: document.updated_at, + record_source: document.record_source, + })) + }) + .await } async fn scoped_reset_legacy_memory( @@ -19391,13 +20529,18 @@ async fn scoped_reset_legacy_memory( )) .into()); } - let counts = api.store.reset_legacy_memory(&path.workspace_id)?; - Ok(Json(server_api::LegacyMemoryResetResponse { - workspace_id: path.workspace_id, - documents_deleted: counts.documents, - staging_records_deleted: counts.staging_records, - staging_resolutions_deleted: counts.staging_resolutions, - })) + api.store + .db_executor() + .run(move || { + let counts = api.store.reset_legacy_memory(&path.workspace_id)?; + Ok(Json(server_api::LegacyMemoryResetResponse { + workspace_id: path.workspace_id, + documents_deleted: counts.documents, + staging_records_deleted: counts.staging_records, + staging_resolutions_deleted: counts.staging_resolutions, + })) + }) + .await } async fn scoped_list_memory_staging( @@ -19405,13 +20548,20 @@ async fn scoped_list_memory_staging( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(list_memory_staging_from_authority( - &api.authority, - query.limit.map(usize::try_from).transpose().map_err(|_| { - Error::InvalidInput("Memory staging limit exceeds this host's usize range".to_string()) - })?, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(list_memory_staging_from_authority( + &api.authority, + query.limit.map(usize::try_from).transpose().map_err(|_| { + Error::InvalidInput( + "Memory staging limit exceeds this host's usize range".to_string(), + ) + })?, + )?)) + }) + .await } async fn scoped_memory_backend_operation( @@ -19420,21 +20570,26 @@ async fn scoped_memory_backend_operation( _context: server_api::ServerRequestContext, Json(operation): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - // The legacy memory document edit is a read/replace/write across store - // calls. Keep that resource atomic without serializing unrelated requests. - let _memory_guard = api - .memory_operation_lock - .lock() - .expect("Memory operation lock poisoned"); - let result = execute_memory_backend_operation_with_authority(&api.authority, operation); - let response = match result { - Ok(result) => MemoryBackendHttpResponse::Ok { result }, - Err(error) => MemoryBackendHttpResponse::Error { - message: sanitize_backend_error(&error.to_string()), - }, - }; - Ok(Json(response)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + // The legacy memory document edit is a read/replace/write across store + // calls. Keep that resource atomic without serializing unrelated requests. + let _memory_guard = api + .memory_operation_lock + .lock() + .expect("Memory operation lock poisoned"); + let result = execute_memory_backend_operation_with_authority(&api.authority, operation); + let response = match result { + Ok(result) => MemoryBackendHttpResponse::Ok { result }, + Err(error) => MemoryBackendHttpResponse::Error { + message: sanitize_backend_error(&error.to_string()), + }, + }; + Ok(Json(response)) + }) + .await } const SUBJEKTIV_SINGLETON_PREFIX: &str = crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX; @@ -19442,6 +20597,66 @@ const SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX: &str = crate::hosts::SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX; const SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE: &str = "subjektiv-memory-consolidation"; +// Runtime callbacks may themselves need DB authority. Synchronous Host workflows +// therefore have a separate finite lane: never monopolize the sole DB worker while +// waiting for Runtime I/O. Existing transaction helpers still finish before I/O. +// Teardown waits must not occupy the ordinary Host lane: a Job may need a +// user stop/control operation on that lane before its Run can finish. Cleanup +// therefore has its own finite owner, still independent of the DB callback lane. +#[derive(Clone, Default)] +struct HostExecutors { + host: Arc>>, + cleanup: Arc>>, +} + +impl HostExecutors { + fn shared() -> Self { + static SHARED: std::sync::OnceLock = std::sync::OnceLock::new(); + SHARED.get_or_init(Self::default).clone() + } + + fn lane( + owner: &std::sync::OnceLock>, + ) -> Result { + owner + .get_or_init(|| { + crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(2).unwrap(), + queued: std::num::NonZeroUsize::new(16).unwrap(), + }) + .ok() + }) + .clone() + .ok_or(Error::DbExecutorUnavailable) + } + + fn host(&self) -> Result { + Self::lane(&self.host) + } + fn cleanup(&self) -> Result { + Self::lane(&self.cleanup) + } +} + +// Some existing lifecycle workflows interleave owned async resource locks with +// synchronous DB/Runtime APIs. Poll the complete workflow on the finite Host +// lane using the original runtime (never create a replacement runtime for its +// cached services). Cancellation releases neither locks nor accepted ownership. +// No SQL transaction contains an await/Runtime call, and this lane is separate +// from the DB worker needed by Runtime callbacks. Do not nest this helper. +async fn run_host_workflow( + executor: crate::db_executor::DbExecutor, + future: impl std::future::Future> + Send + 'static, +) -> std::result::Result +where + T: Send + 'static, + E: From + Send + 'static, +{ + let runtime = + tokio::runtime::Handle::try_current().map_err(|_| E::from(Error::DbExecutorUnavailable))?; + executor.run(move || runtime.block_on(future)).await +} + fn open_subjektiv_store(api: &WorkspaceApi) -> ApiResult { crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) .map_err(|error| ApiError::from(Error::Store(error.to_string()))) @@ -19510,11 +20725,26 @@ fn require_dedicated_subjektiv_consolidation_worker( } } -fn subjektiv_subject_scope( +// Store-only authority discovery. A mandatory Session attribution callback must +// reject consolidation roles without consulting Runtime: its create/restore parent +// can already hold a Host worker while Runtime waits for that callback to finish. +enum SubjektivStoredScopeAuthority { + Subject, + JobConsolidation, + LegacyConsolidation { singleton_key: String }, +} + +struct SubjektivStoredWorkerScope { + subject_id: String, + worker: RuntimeWorkerRef, + authority: SubjektivStoredScopeAuthority, +} + +fn subjektiv_worker_scope_from_store( api: &WorkspaceApi, workspace_id: &str, context: &server_api::ServerRequestContext, -) -> ApiResult<(String, RuntimeWorkerRef, SubjektivWorkerAuthority)> { +) -> ApiResult { let source = context.runtime_source.as_ref().ok_or_else(|| { Error::WorkspacePermissionDenied( "subjektiv operations require authenticated Runtime-owned source authority".to_string(), @@ -19527,11 +20757,11 @@ fn subjektiv_subject_scope( })?; let worker = RuntimeWorkerRef::new(&source.runtime_id, worker_id); if let Some(grant) = subjektiv_job::active_grant(api, workspace_id, &worker)? { - return Ok(( - grant.subject_id, + return Ok(SubjektivStoredWorkerScope { + subject_id: grant.subject_id, worker, - SubjektivWorkerAuthority::Consolidation, - )); + authority: SubjektivStoredScopeAuthority::JobConsolidation, + }); } let lease = api .store @@ -19542,15 +20772,47 @@ fn subjektiv_subject_scope( ) })?; let (subject_id, authority) = subjektiv_scope_from_singleton_key(&lease.key)?; - if authority == SubjektivWorkerAuthority::Consolidation { - let mut summary = api - .runtime - .worker(&worker) - .map_err(|error| error.into_error())?; - summary.singleton_key = Some(lease.key.clone()); - require_dedicated_subjektiv_consolidation_worker(&summary, workspace_id, &lease.key)?; - } - Ok((subject_id.to_string(), worker, authority)) + let authority = match authority { + SubjektivWorkerAuthority::Subject => SubjektivStoredScopeAuthority::Subject, + SubjektivWorkerAuthority::Consolidation => { + SubjektivStoredScopeAuthority::LegacyConsolidation { + singleton_key: lease.key.clone(), + } + } + }; + Ok(SubjektivStoredWorkerScope { + subject_id: subject_id.to_owned(), + worker, + authority, + }) +} + +// Mixed callers run this complete policy boundary on Host. Only legacy +// consolidation authority needs synchronous Runtime profile/tag verification. +fn subjektiv_subject_scope( + api: &WorkspaceApi, + workspace_id: &str, + context: &server_api::ServerRequestContext, +) -> ApiResult<(String, RuntimeWorkerRef, SubjektivWorkerAuthority)> { + let scope = subjektiv_worker_scope_from_store(api, workspace_id, context)?; + let authority = match scope.authority { + SubjektivStoredScopeAuthority::Subject => SubjektivWorkerAuthority::Subject, + SubjektivStoredScopeAuthority::JobConsolidation => SubjektivWorkerAuthority::Consolidation, + SubjektivStoredScopeAuthority::LegacyConsolidation { singleton_key } => { + let mut summary = api + .runtime + .worker(&scope.worker) + .map_err(|error| error.into_error())?; + summary.singleton_key = Some(singleton_key.clone()); + require_dedicated_subjektiv_consolidation_worker( + &summary, + workspace_id, + &singleton_key, + )?; + SubjektivWorkerAuthority::Consolidation + } + }; + Ok((scope.subject_id, scope.worker, authority)) } fn require_subjektiv_worker_authority( @@ -19574,8 +20836,8 @@ fn subjektiv_session_attribution( context: &server_api::ServerRequestContext, session_id: &str, ) -> ApiResult { - let (subject_id, worker, authority) = subjektiv_subject_scope(api, workspace_id, context)?; - if authority != SubjektivWorkerAuthority::Subject { + let scope = subjektiv_worker_scope_from_store(api, workspace_id, context)?; + if !matches!(scope.authority, SubjektivStoredScopeAuthority::Subject) { return Err(Error::WorkspacePermissionDenied( "subject Session attribution requires the current subject Worker".into(), ) @@ -19586,9 +20848,9 @@ fn subjektiv_session_attribution( return Err(Error::InvalidInput("committed session_id is invalid".to_string()).into()); } crate::subjektiv::SubjectSessionAttribution::new( - subject_id, - worker.runtime_id, - worker.worker_id, + scope.subject_id, + scope.worker.runtime_id, + scope.worker.worker_id, session_id, ) .map_err(|error| Error::InvalidInput(error.to_string()).into()) @@ -19659,18 +20921,23 @@ async fn scoped_create_subjektiv_subject( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = crate::subjektiv::SubjectRole::new(request.role) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .create_subject_with_behavior(role, request.behavior_md) - .map_err(subjektiv_store_error)?; - Ok(( - StatusCode::CREATED, - Json(subjektiv_subject_response(&api, subject)?), - )) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let role = crate::subjektiv::SubjectRole::new(request.role) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .create_subject_with_behavior(role, request.behavior_md) + .map_err(subjektiv_store_error)?; + Ok(( + StatusCode::CREATED, + Json(subjektiv_subject_response(&api, subject)?), + )) + }) + .await } #[derive(Debug, Serialize, Deserialize)] @@ -19686,76 +20953,86 @@ async fn scoped_list_subjektiv_subjects( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let limit = query.limit.unwrap_or(20); - if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { - return Err(Error::InvalidInput(format!( - "subject list limit must be within 1..={}", - server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT - )) - .into()); - } - let cursor = query - .cursor - .as_deref() - .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) - .transpose()?; - if cursor - .as_ref() - .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) - { - return Err(Error::InvalidInput( - "subjektiv subject cursor does not match Workspace".to_string(), - ) - .into()); - } - let page = open_subjektiv_store(&api)? - .list_subjects_after( - limit, - cursor + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let limit = query.limit.unwrap_or(20); + if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { + return Err(Error::InvalidInput(format!( + "subject list limit must be within 1..={}", + server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT + )) + .into()); + } + let cursor = query + .cursor + .as_deref() + .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) + .transpose()?; + if cursor .as_ref() - .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), - ) - .map_err(subjektiv_store_error)?; - let next_cursor = if page.has_more { - page.items - .last() - .map(|subject| { - encode_subjektiv_cursor( - "subjects", - &SubjektivSubjectCursor { - workspace_id: path.workspace_id.clone(), - created_at: subject.created_at.clone(), - subject_id: subject.id.clone(), - }, + .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) + { + return Err(Error::InvalidInput( + "subjektiv subject cursor does not match Workspace".to_string(), ) - }) - .transpose()? - } else { - None - }; - Ok(Json(server_api::SubjektivSubjectListResponse { - limit, - items: page - .items - .into_iter() - .map(|subject| subjektiv_subject_response(&api, subject)) - .collect::>>()?, - next_cursor, - has_more: page.has_more, - })) + .into()); + } + let page = open_subjektiv_store(&api)? + .list_subjects_after( + limit, + cursor + .as_ref() + .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), + ) + .map_err(subjektiv_store_error)?; + let next_cursor = if page.has_more { + page.items + .last() + .map(|subject| { + encode_subjektiv_cursor( + "subjects", + &SubjektivSubjectCursor { + workspace_id: path.workspace_id.clone(), + created_at: subject.created_at.clone(), + subject_id: subject.id.clone(), + }, + ) + }) + .transpose()? + } else { + None + }; + Ok(Json(server_api::SubjektivSubjectListResponse { + limit, + items: page + .items + .into_iter() + .map(|subject| subjektiv_subject_response(&api, subject)) + .collect::>>()?, + next_cursor, + has_more: page.has_more, + })) + }) + .await } async fn scoped_get_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .subject(&path.subject_id) - .map_err(|error| Error::Store(error.to_string()))? - .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .subject(&path.subject_id) + .map_err(|error| Error::Store(error.to_string()))? + .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } async fn scoped_update_subjektiv_subject_behavior( @@ -19763,17 +21040,22 @@ async fn scoped_update_subjektiv_subject_behavior( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .update_subject_behavior( - &path.subject_id, - &request.expected_behavior_md, - request.behavior_md, - ) - .map_err(subjektiv_store_error)?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .update_subject_behavior( + &path.subject_id, + &request.expected_behavior_md, + request.behavior_md, + ) + .map_err(subjektiv_store_error)?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } fn require_subjektiv_subject( @@ -19794,47 +21076,53 @@ async fn scoped_get_subjektiv_resident_surface( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - let resident = store - .resident_surface(&path.subject_id) - .map_err(subjektiv_store_error)?; - let availability = match resident.availability { - crate::subjektiv::SurfaceAvailability::Ungenerated => { - server_api::SubjektivResidentSurfaceAvailability::Ungenerated - } - crate::subjektiv::SurfaceAvailability::Stale => { - server_api::SubjektivResidentSurfaceAvailability::Stale - } - crate::subjektiv::SurfaceAvailability::Failed => { - server_api::SubjektivResidentSurfaceAvailability::Failed - } - crate::subjektiv::SurfaceAvailability::Ready => { - server_api::SubjektivResidentSurfaceAvailability::Ready - } - }; - let snapshot = resident - .snapshot - .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { - snapshot_id: snapshot.id, - body_md: snapshot.body_md, - memory_refs: snapshot - .memory_refs - .into_iter() - .map(|reference| server_api::SubjektivMemoryChangeRef { - memory_id: reference.memory_id, - change_id: reference.change_id, - }) - .collect(), - built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, - created_at: snapshot.created_at, - }); - Ok(Json(server_api::SubjektivResidentSurfaceResponse { - subject_id: path.subject_id, - availability, - snapshot, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + let resident = store + .resident_surface(&path.subject_id) + .map_err(subjektiv_store_error)?; + let availability = match resident.availability { + crate::subjektiv::SurfaceAvailability::Ungenerated => { + server_api::SubjektivResidentSurfaceAvailability::Ungenerated + } + crate::subjektiv::SurfaceAvailability::Stale => { + server_api::SubjektivResidentSurfaceAvailability::Stale + } + crate::subjektiv::SurfaceAvailability::Failed => { + server_api::SubjektivResidentSurfaceAvailability::Failed + } + crate::subjektiv::SurfaceAvailability::Ready => { + server_api::SubjektivResidentSurfaceAvailability::Ready + } + }; + let snapshot = + resident + .snapshot + .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { + snapshot_id: snapshot.id, + body_md: snapshot.body_md, + memory_refs: snapshot + .memory_refs + .into_iter() + .map(|reference| server_api::SubjektivMemoryChangeRef { + memory_id: reference.memory_id, + change_id: reference.change_id, + }) + .collect(), + built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, + created_at: snapshot.created_at, + }); + Ok(Json(server_api::SubjektivResidentSurfaceResponse { + subject_id: path.subject_id, + availability, + snapshot, + })) + }) + .await } async fn scoped_list_subjektiv_memories( @@ -19842,14 +21130,19 @@ async fn scoped_list_subjektiv_memories( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_query( - &store, - &path.subject_id, - query, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_query( + &store, + &path.subject_id, + query, + )?)) + }) + .await } async fn scoped_get_subjektiv_memory( @@ -19857,21 +21150,26 @@ async fn scoped_get_subjektiv_memory( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_read( - &store, - &path.subject_id, - server_api::SubjektivMemoryReadRequest { - memory_id: path.memory_id, - change_id: query.change_id, - offset: query.offset, - byte_offset: query.byte_offset, - limit: query.limit, - evidence_cursor: query.evidence_cursor, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_read( + &store, + &path.subject_id, + server_api::SubjektivMemoryReadRequest { + memory_id: path.memory_id, + change_id: query.change_id, + offset: query.offset, + byte_offset: query.byte_offset, + limit: query.limit, + evidence_cursor: query.evidence_cursor, + }, + )?)) + }) + .await } async fn scoped_list_subjektiv_memory_changes( @@ -19879,31 +21177,41 @@ async fn scoped_list_subjektiv_memory_changes( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_list_changes( - &store, - &path.subject_id, - server_api::SubjektivMemoryListChangesRequest { - memory_id: path.memory_id, - limit: query.limit, - cursor: query.cursor, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_list_changes( + &store, + &path.subject_id, + server_api::SubjektivMemoryListChangesRequest { + memory_id: path.memory_id, + limit: query.limit, + cursor: query.cursor, + }, + )?)) + }) + .await } async fn scoped_retire_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .retire_subject(&path.subject_id) - .map_err(subjektiv_store_error)?; - // Retirement deliberately does not stop or remove the current Worker. The - // Worker lease and subject lifecycle are independent authorities. - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .retire_subject(&path.subject_id) + .map_err(subjektiv_store_error)?; + // Retirement deliberately does not stop or remove the current Worker. The + // Worker lease and subject lifecycle are independent authorities. + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } async fn scoped_stage_subjektiv_candidate( @@ -19912,58 +21220,63 @@ async fn scoped_stage_subjektiv_candidate( context: server_api::ServerRequestContext, Json(mut request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )?; - let subject_id = attribution.subject_id.clone(); - let session_id = attribution.session_id.as_str(); - for source_ref in &mut request.operation.source_refs { - if source_ref - .session_id - .as_deref() - .is_some_and(|value| value != session_id) - { - return Err(Error::InvalidInput( - "candidate source_ref belongs to a different Session".to_string(), - ) - .into()); - } - source_ref.session_id = Some(session_id.to_string()); - } - if matches!( - request.operation.candidate.kind, - memory::extract::CandidateKind::Preference - ) && request.operation.evidence.iter().any(|evidence| { - !matches!( - evidence.origin.as_ref().map(|origin| &origin.kind), - Some(memory::schema::EvidenceOriginKind::HumanInput) - ) - }) { - return Err(Error::InvalidInput( - "preference candidates require exclusively HumanInput evidence".to_string(), - ) - .into()); - } - let candidate_id = Uuid::now_v7().to_string(); - let record = memory::extract::StagingRecord::from_candidate( - candidate_id, - request.operation.extract_run_id, - request.operation.source, - request.operation.candidate, - request.operation.evidence, - request.operation.source_refs, - ); - let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); - let (staged, _) = open_subjektiv_store(&api)? - .stage_candidate_with_attribution(record, attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - Ok(Json(server_api::SubjektivStageCandidateResponse { - staging_id: staged.id, - })) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )?; + let subject_id = attribution.subject_id.clone(); + let session_id = attribution.session_id.as_str(); + for source_ref in &mut request.operation.source_refs { + if source_ref + .session_id + .as_deref() + .is_some_and(|value| value != session_id) + { + return Err(Error::InvalidInput( + "candidate source_ref belongs to a different Session".to_string(), + ) + .into()); + } + source_ref.session_id = Some(session_id.to_string()); + } + if matches!( + request.operation.candidate.kind, + memory::extract::CandidateKind::Preference + ) && request.operation.evidence.iter().any(|evidence| { + !matches!( + evidence.origin.as_ref().map(|origin| &origin.kind), + Some(memory::schema::EvidenceOriginKind::HumanInput) + ) + }) { + return Err(Error::InvalidInput( + "preference candidates require exclusively HumanInput evidence".to_string(), + ) + .into()); + } + let candidate_id = Uuid::now_v7().to_string(); + let record = memory::extract::StagingRecord::from_candidate( + candidate_id, + request.operation.extract_run_id, + request.operation.source, + request.operation.candidate, + request.operation.evidence, + request.operation.source_refs, + ); + let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); + let (staged, _) = open_subjektiv_store(&api)? + .stage_candidate_with_attribution(record, attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + Ok(Json(server_api::SubjektivStageCandidateResponse { + staging_id: staged.id, + })) + }) + .await } async fn scoped_record_subjektiv_session( @@ -19972,25 +21285,37 @@ async fn scoped_record_subjektiv_session( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = if request.create_if_missing { - let attribution = - subjektiv_session_attribution(&api, &path.workspace_id, &context, &request.session_id)?; - open_subjektiv_store(&api)? - .record_session_attribution(attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))? - } else { - require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )? - }; - Ok(Json(server_api::SubjektivRecordSessionResponse { - subject_id: attribution.subject_id, - session_id: attribution.session_id, - })) + // Runtime create/restore waits for this mandatory callback while holding + // Host capacity. Its Subject-only authority and whole attribution SQL must + // use the independent DB lane, never the parent's waiting Host lane. + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = if request.create_if_missing { + let attribution = subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )?; + open_subjektiv_store(&api)? + .record_session_attribution(attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))? + } else { + require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )? + }; + Ok(Json(server_api::SubjektivRecordSessionResponse { + subject_id: attribution.subject_id, + session_id: attribution.session_id, + })) + }) + .await } const SUBJEKTIV_SESSION_DEFAULT_LIMIT: usize = 20; @@ -20084,50 +21409,61 @@ async fn scoped_subjektiv_session_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, controller, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - require_subjektiv_worker_authority( - authority, - SubjektivWorkerAuthority::Subject, - "Session history read", - )?; - let store = match open_subjektiv_store(&api) { - Ok(store) => store, - Err(error) => { - return Ok(Json(subjektiv_session_error( - server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, - error.error.to_string(), - ))); - } - }; - let response = match request.operation { - server_api::SubjektivSessionBackendOperation::List(input) => subjektiv_session_list( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - server_api::SubjektivSessionBackendOperation::Search(input) => subjektiv_session_search( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - server_api::SubjektivSessionBackendOperation::Read(input) => subjektiv_session_read( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - }; - Ok(Json(response)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, controller, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + require_subjektiv_worker_authority( + authority, + SubjektivWorkerAuthority::Subject, + "Session history read", + )?; + let store = match open_subjektiv_store(&api) { + Ok(store) => store, + Err(error) => { + return Ok(Json(subjektiv_session_error( + server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, + error.error.to_string(), + ))); + } + }; + let response = match request.operation { + server_api::SubjektivSessionBackendOperation::List(input) => { + subjektiv_session_list( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + server_api::SubjektivSessionBackendOperation::Search(input) => { + subjektiv_session_search( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + server_api::SubjektivSessionBackendOperation::Read(input) => { + subjektiv_session_read( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + }; + Ok(Json(response)) + }) + .await } fn subjektiv_session_list( @@ -21318,13 +22654,10 @@ async fn scoped_subjektiv_memory_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, source_worker, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - let store = open_subjektiv_store(&api)?; use server_api::SubjektivMemoryBackendOperation as Op; - // Authentication, live Worker/Job authority and committed Session provenance - // remain Backend-owned. Only the domain operation/projection is shared. + // Subject-only operations are complete DB boundaries. This includes resident + // context callbacks during Runtime create/restore; they must not queue behind + // the Host parents waiting for them. Other roles can need legacy Runtime checks. let (expected, label) = match &request.operation { Op::ResidentSummary(_) => ( Some(SubjektivWorkerAuthority::Subject), @@ -21372,61 +22705,93 @@ async fn scoped_subjektiv_memory_backend( ), Op::Query(_) | Op::Read(_) | Op::ListChanges(_) => (None, "recall"), }; - if let Some(expected) = expected { - require_subjektiv_worker_authority(authority, expected, label)?; - } - let host_context = match authority { - SubjektivWorkerAuthority::Subject => { - let attribution = if let Op::StageExplicit(input) = &request.operation { - Some(require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &input.session_id, - )?) - } else { - None - }; - HostOperationContext::validated_body(&store, &subject_id, attribution)? - } - SubjektivWorkerAuthority::Consolidation => { - match &request.operation { - Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - _ => {} - } - let grant = subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; - let job = api - .store - .worker_registry_projection(&path.workspace_id, &source_worker)? - .and_then(|projection| projection.job) - .map(|binding| JobAttemptBinding { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - }); - HostOperationContext::validated_consolidation( - &store, - &subject_id, - grant.map(|g| g.candidate_ids), - job, - )? - } + let executor = if expected == Some(SubjektivWorkerAuthority::Subject) { + api.store.db_executor() + } else { + api.host_executors.host()? }; - Ok(Json(subjektiv::api::execute( - &store, - &host_context, - request.operation, - )?)) + executor + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, source_worker, authority) = if expected + == Some(SubjektivWorkerAuthority::Subject) + { + let scope = subjektiv_worker_scope_from_store(&api, &path.workspace_id, &context)?; + if !matches!(scope.authority, SubjektivStoredScopeAuthority::Subject) { + return Err(Error::WorkspacePermissionDenied(format!( + "subjektiv {label} is not authorized for this Worker role" + )) + .into()); + } + ( + scope.subject_id, + scope.worker, + SubjektivWorkerAuthority::Subject, + ) + } else { + subjektiv_subject_scope(&api, &path.workspace_id, &context)? + }; + let store = open_subjektiv_store(&api)?; + // Authentication, live Worker/Job authority and committed Session provenance + // remain Backend-owned. Only the domain operation/projection is shared. + if let Some(expected) = expected { + require_subjektiv_worker_authority(authority, expected, label)?; + } + let host_context = match authority { + SubjektivWorkerAuthority::Subject => { + let attribution = if let Op::StageExplicit(input) = &request.operation { + Some(require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &input.session_id, + )?) + } else { + None + }; + HostOperationContext::validated_body(&store, &subject_id, attribution)? + } + SubjektivWorkerAuthority::Consolidation => { + match &request.operation { + Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + _ => {} + } + let grant = + subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; + let job = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|projection| projection.job) + .map(|binding| JobAttemptBinding { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + }); + HostOperationContext::validated_consolidation( + &store, + &subject_id, + grant.map(|g| g.candidate_ids), + job, + )? + } + }; + Ok(Json(subjektiv::api::execute( + &store, + &host_context, + request.operation, + )?)) + }) + .await } fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error { @@ -21463,13 +22828,22 @@ async fn scoped_subjektiv_memory_consolidation( context: server_api::ServerRequestContext, Json(operation): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, _, authority) = subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - require_subjektiv_worker_authority( - authority, - SubjektivWorkerAuthority::Subject, - "consolidation request", - )?; + let scope_api = api.clone(); + let subject_id = api + .host_executors + .host()? + .run(move || { + validate_workspace_scope(&scope_api, &path.workspace_id)?; + let (subject_id, _, authority) = + subjektiv_subject_scope(&scope_api, &path.workspace_id, &context)?; + require_subjektiv_worker_authority( + authority, + SubjektivWorkerAuthority::Subject, + "consolidation request", + )?; + Ok::<_, ApiError>(subject_id) + }) + .await?; Ok(Json( start_subjektiv_staging_consolidation(api, &subject_id, operation).await?, )) @@ -21530,10 +22904,17 @@ async fn start_subjektiv_staging_consolidation( subject_id: &str, operation: MemoryConsolidateStagingOperation, ) -> ApiResult { - let store = open_subjektiv_store(&api)?; - let (candidate_count, total_bytes) = store - .pending_staging_backlog(subject_id) - .map_err(subjektiv_store_error)?; + let job_api = api.clone(); + let subject = subject_id.to_owned(); + let (candidate_count, total_bytes) = api + .store + .db_executor() + .run(move || { + open_subjektiv_store(&job_api)? + .pending_staging_backlog(&subject) + .map_err(|error| ApiError::from(subjektiv_store_error(error))) + }) + .await?; if !subjektiv_job::retire_legacy_worker(&api, subject_id).await? { return Ok(MemoryConsolidationOutput { status: "skipped_legacy_inflight".into(), @@ -21541,10 +22922,18 @@ async fn start_subjektiv_staging_consolidation( candidate_count, total_bytes, }); } - let surface_availability = store - .resident_surface(subject_id) - .map_err(subjektiv_store_error)? - .availability; + let job_api = api.clone(); + let subject = subject_id.to_owned(); + let surface_availability = api + .store + .db_executor() + .run(move || { + open_subjektiv_store(&job_api)? + .resident_surface(&subject) + .map(|surface| surface.availability) + .map_err(|error| ApiError::from(subjektiv_store_error(error))) + }) + .await?; if !subject_consolidation_required( candidate_count, total_bytes, @@ -21571,7 +22960,11 @@ async fn start_subjektiv_staging_consolidation( }); } - subjektiv_job::dispatch(&api, subject_id, candidate_count, total_bytes) + let subject = subject_id.to_owned(); + api.host_executors + .host()? + .run(move || subjektiv_job::dispatch(&api, &subject, candidate_count, total_bytes)) + .await } async fn scoped_memory_consolidation( @@ -21579,8 +22972,13 @@ async fn scoped_memory_consolidation( AxumPath(path): AxumPath, Json(operation): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(start_memory_staging_consolidation(api, operation)?)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(start_memory_staging_consolidation(api, operation)?)) + }) + .await } fn start_memory_staging_consolidation( @@ -21780,72 +23178,92 @@ async fn scoped_list_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::catalog(&state).map(Json).map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::catalog(&state).map(Json).map_err(skill_api_error) + }) + .await } async fn scoped_lint_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::lint(&state).map(Json).map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::lint(&state).map(Json).map_err(skill_api_error) + }) + .await } async fn scoped_get_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::detail(&state, &path.name) - .map(Json) - .map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::detail(&state, &path.name) + .map(Json) + .map_err(skill_api_error) + }) + .await } async fn scoped_activate_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::activation(&state, &path.name) - .map(Json) - .map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::activation(&state, &path.name) + .map(Json) + .map_err(skill_api_error) + }) + .await } fn skill_api_error(error: skills::SkillError) -> ApiError { @@ -21895,8 +23313,13 @@ async fn scoped_query_objectives( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_objectives(query)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_objectives(query)?)) + }) + .await } async fn scoped_show_objective( @@ -21904,10 +23327,15 @@ async fn scoped_show_objective( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority.show_objective(&path.objective_id, query)?, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority.show_objective(&path.objective_id, query)?, + )) + }) + .await } async fn scoped_create_objective( @@ -21915,15 +23343,20 @@ async fn scoped_create_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.create_objective( - ObjectiveCreateInput { - title: request.title, - body_md: request.body_md, - state: request.state, - linked_tickets: request.linked_tickets, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.create_objective( + ObjectiveCreateInput { + title: request.title, + body_md: request.body_md, + state: request.state, + linked_tickets: request.linked_tickets, + }, + )?)) + }) + .await } async fn scoped_edit_objective( @@ -21931,16 +23364,21 @@ async fn scoped_edit_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.edit_objective( - &path.objective_id, - ObjectiveEditInput { - title: request.title, - old_string: request.old_string, - new_string: request.new_string, - replace_all: request.replace_all, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.edit_objective( + &path.objective_id, + ObjectiveEditInput { + title: request.title, + old_string: request.old_string, + new_string: request.new_string, + replace_all: request.replace_all, + }, + )?)) + }) + .await } async fn scoped_set_objective_state( @@ -21948,11 +23386,16 @@ async fn scoped_set_objective_state( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority - .set_objective_state(&path.objective_id, &request.state)?, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority + .set_objective_state(&path.objective_id, &request.state)?, + )) + }) + .await } async fn scoped_link_objective_ticket( @@ -21960,22 +23403,32 @@ async fn scoped_link_objective_ticket( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.link_objective_ticket( - &path.objective_id, - &request.ticket_id, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.link_objective_ticket( + &path.objective_id, + &request.ticket_id, + )?)) + }) + .await } async fn scoped_unlink_objective_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.unlink_objective_ticket( - &path.objective_id, - &path.ticket_id, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.unlink_objective_ticket( + &path.objective_id, + &path.ticket_id, + )?)) + }) + .await } async fn scoped_list_repositories( @@ -21993,56 +23446,61 @@ async fn scoped_create_repository( Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { require_workspace_owner(&api, &path.workspace_id, &actor, "ManageRepositories").await?; + api.store + .db_executor() + .run(move || { + server_api::validate_repository_key(&request.repository_key) + .map_err(|error| Error::InvalidInput(format!("invalid Repository key: {error}")))?; + let repository_key = request.repository_key; + let default_ref = request + .default_ref + .as_deref() + .map(|value| normalize_repository_text(value, "default_ref", 512)) + .transpose()?; + let source = parse_repository_source(&request.source).map_err(|error| { + Error::InvalidInput(format!("invalid repository source: {error}")) + })?; + let now = Utc::now().to_rfc3339(); + let record = RepositoryRecord { + workspace_id: path.workspace_id.clone(), + repository_id: Uuid::now_v7().to_string(), + repository_key: repository_key.clone(), + kind: "git".to_string(), + provider: Some("git".to_string()), + source_fingerprint: repository_source_fingerprint(&source), + source, + default_ref, + observed_status: server_api::RepositoryObservedStatus::Unverified, + observed_at: None, + created_at: now.clone(), + updated_at: now, + }; - server_api::validate_repository_key(&request.repository_key) - .map_err(|error| Error::InvalidInput(format!("invalid Repository key: {error}")))?; - let repository_key = request.repository_key; - let default_ref = request - .default_ref - .as_deref() - .map(|value| normalize_repository_text(value, "default_ref", 512)) - .transpose()?; - let source = parse_repository_source(&request.source) - .map_err(|error| Error::InvalidInput(format!("invalid repository source: {error}")))?; - let now = Utc::now().to_rfc3339(); - let record = RepositoryRecord { - workspace_id: path.workspace_id.clone(), - repository_id: Uuid::now_v7().to_string(), - repository_key: repository_key.clone(), - kind: "git".to_string(), - provider: Some("git".to_string()), - source_fingerprint: repository_source_fingerprint(&source), - source, - default_ref, - observed_status: server_api::RepositoryObservedStatus::Unverified, - observed_at: None, - created_at: now.clone(), - updated_at: now, - }; - - let (status, replayed) = match api.config_store.insert_repository(&record)? { - RepositoryInsertOutcome::Created => (StatusCode::CREATED, false), - RepositoryInsertOutcome::Existing(existing) - if repository_create_intent_matches(&existing, &record) => - { - (StatusCode::OK, true) - } - RepositoryInsertOutcome::Existing(_) => { - return Err(Error::RepositoryConflict(format!( + let (status, replayed) = match api.config_store.insert_repository(&record)? { + RepositoryInsertOutcome::Created => (StatusCode::CREATED, false), + RepositoryInsertOutcome::Existing(existing) + if repository_create_intent_matches(&existing, &record) => + { + (StatusCode::OK, true) + } + RepositoryInsertOutcome::Existing(_) => { + return Err(Error::RepositoryConflict(format!( "Repository key {repository_key} already exists with different registration intent" )) .into()); - } - }; + } + }; - Ok(( - status, - Json(CreateWorkspaceRepositoryResponse { - workspace_id: path.workspace_id, - repository_key, - replayed, - }), - )) + Ok(( + status, + Json(CreateWorkspaceRepositoryResponse { + workspace_id: path.workspace_id, + repository_key, + replayed, + }), + )) + }) + .await } fn normalize_repository_text(value: &str, field: &str, max_len: usize) -> Result { @@ -22084,7 +23542,10 @@ async fn scoped_probe_repository_ssh_connection( ) -> ApiResult> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; require_active_workspace_runtime_binding(&api, &request.runtime_id).await?; - probe_repository_ssh_connection(&api, &path, &request.runtime_id).map(Json) + api.host_executors + .host()? + .run(move || probe_repository_ssh_connection(&api, &path, &request.runtime_id).map(Json)) + .await } async fn scoped_confirm_repository_ssh_host_trust( @@ -22095,35 +23556,40 @@ async fn scoped_confirm_repository_ssh_host_trust( ) -> ApiResult> { require_manage_repository_secrets(&api, &path.workspace_id, &actor).await?; require_active_workspace_runtime_binding(&api, &request.runtime_id).await?; - let probe = probe_repository_ssh_connection(&api, &path, &request.runtime_id)?; - let candidate = probe - .candidates - .iter() - .find(|candidate| candidate.host_key == request.host_key) - .ok_or_else(|| { - settings_bad_request( - "repository_ssh_host_key_not_observed", - "Selected SSH host key was not presented by the target Runtime", - ) - })?; - if request.expected_host_key_fingerprint != probe.expected_host_key_fingerprint { - return Err(ApiError::from(Error::WorkspaceConfigConflict( - "SSH host trust fingerprint changed after the connection test".to_string(), - ))); - } - let host_trust = api.repository_secrets.put_host_trust( - &path.workspace_id, - PutRepositorySshHostTrustRequest { - operation_id: request.operation_id, - host_trust_id: probe.host_trust_id, - hostname: probe.hostname, - port: probe.port, - host_key: candidate.host_key.clone(), - expected_fingerprint: probe.expected_host_key_fingerprint, - }, - &actor.account_id, - )?; - Ok(Json(host_trust)) + api.host_executors + .host()? + .run(move || { + let probe = probe_repository_ssh_connection(&api, &path, &request.runtime_id)?; + let candidate = probe + .candidates + .iter() + .find(|candidate| candidate.host_key == request.host_key) + .ok_or_else(|| { + settings_bad_request( + "repository_ssh_host_key_not_observed", + "Selected SSH host key was not presented by the target Runtime", + ) + })?; + if request.expected_host_key_fingerprint != probe.expected_host_key_fingerprint { + return Err(ApiError::from(Error::WorkspaceConfigConflict( + "SSH host trust fingerprint changed after the connection test".to_string(), + ))); + } + let host_trust = api.repository_secrets.put_host_trust( + &path.workspace_id, + PutRepositorySshHostTrustRequest { + operation_id: request.operation_id, + host_trust_id: probe.host_trust_id, + hostname: probe.hostname, + port: probe.port, + host_key: candidate.host_key.clone(), + expected_fingerprint: probe.expected_host_key_fingerprint, + }, + &actor.account_id, + )?; + Ok(Json(host_trust)) + }) + .await } fn repository_ssh_connection_trust_state( @@ -22483,6 +23949,9 @@ async fn worker_remove_contract( .map_err(|error| { let status = match error { crate::worker_source::WorkerMutationSourceProofError::Replay => StatusCode::CONFLICT, + crate::worker_source::WorkerMutationSourceProofError::Unavailable => { + StatusCode::SERVICE_UNAVAILABLE + } crate::worker_source::WorkerMutationSourceProofError::Authority(_) => { StatusCode::INTERNAL_SERVER_ERROR } @@ -22497,7 +23966,8 @@ async fn worker_remove_contract( ) })?; let worker = - WorkspaceWorker::resolve(api, &request.target_runtime_id, &request.target_worker_id) + WorkspaceWorker::resolve_async(api, &request.target_runtime_id, &request.target_worker_id) + .await .map_err(|error| match error { Error::UnknownWorker { .. } => server_api::WorkerRemoveApiError::new( StatusCode::NOT_FOUND.as_u16(), @@ -22553,22 +24023,28 @@ async fn scoped_get_workspace_worker( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker_ref = api - .store - .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? - .ok_or_else(|| Error::UnknownWorker { - worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), - })?; - let workers = workers_response(api.clone())?; - workers - .items - .into_iter() - .find(|worker| { - worker.runtime_id == worker_ref.runtime_id && worker.worker_id == worker_ref.worker_id + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let worker_ref = api + .store + .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? + .ok_or_else(|| Error::UnknownWorker { + worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), + })?; + let workers = workers_response(api.clone())?; + workers + .items + .into_iter() + .find(|worker| { + worker.runtime_id == worker_ref.runtime_id + && worker.worker_id == worker_ref.worker_id + }) + .map(Json) + .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) }) - .map(Json) - .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) + .await } async fn discover_workspace_workers( @@ -22600,7 +24076,11 @@ async fn discover_workspace_workers( "invalid_workspace_worker_discovery_cursor".to_string(), )) })?; - let workers = workers_response(api)?; + let workers = api + .store + .db_executor() + .run(move || workers_response(api)) + .await?; Ok(workspace_worker_discovery_page( workers.items, query, @@ -22691,8 +24171,13 @@ async fn scoped_workspace_orchestrator_status( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(workspace_orchestrator_response(&api, "observed"))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(workspace_orchestrator_response(&api, "observed"))) + }) + .await } async fn list_known_workers( @@ -22700,36 +24185,43 @@ async fn list_known_workers( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let limit = api.config.max_records.clamp(1, 500); - let grants = - api.store - .list_active_worker_control_grants(&path.workspace_id, &controller, limit + 1)?; - let truncated = grants.len() > limit; - let mut items = Vec::with_capacity(grants.len().min(limit)); - for grant in grants.into_iter().take(limit) { - let summary = api - .runtime - .worker(&grant.subject) - .map_err(|error| error.into_error())?; - items.push(server_api::WorkerControlRecord { - subject: server_api::WorkerControlSubject { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - relation: grant.relation, - origin: grant.origin, - permissions: grant.permissions, - summary: worker_launch_worker_summary(summary), - }); - } - Ok(Json(server_api::WorkerControlListResponse { - workspace_id: path.workspace_id, - items, - truncated, - })) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let limit = api.config.max_records.clamp(1, 500); + let grants = api.store.list_active_worker_control_grants( + &path.workspace_id, + &controller, + limit + 1, + )?; + let truncated = grants.len() > limit; + let mut items = Vec::with_capacity(grants.len().min(limit)); + for grant in grants.into_iter().take(limit) { + let summary = api + .runtime + .worker(&grant.subject) + .map_err(|error| error.into_error())?; + items.push(server_api::WorkerControlRecord { + subject: server_api::WorkerControlSubject { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + relation: grant.relation, + origin: grant.origin, + permissions: grant.permissions, + summary: worker_launch_worker_summary(summary), + }); + } + Ok(Json(server_api::WorkerControlListResponse { + workspace_id: path.workspace_id, + items, + truncated, + })) + }) + .await } fn scoped_worker_control_operation_id(controller: &RuntimeWorkerRef, operation_id: &str) -> String { @@ -22750,6 +24242,27 @@ async fn spawn_known_worker( } async fn spawn_known_worker_authenticated( + State(api): State, + AxumPath(path): AxumPath, + headers: HeaderMap, + Json(request): Json, + context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + spawn_known_worker_authenticated_on_host( + State(api), + AxumPath(path), + headers, + Json(request), + context.as_ref(), + ) + .await + }) + .await +} + +async fn spawn_known_worker_authenticated_on_host( State(api): State, AxumPath(path): AxumPath, headers: HeaderMap, @@ -22792,7 +24305,7 @@ async fn spawn_known_worker_authenticated( operation_id: scoped_worker_control_operation_id(&controller, &operation_id), input_fingerprint, }); - let response = create_workspace_worker_authenticated( + let response = create_workspace_worker_authenticated_on_host( api.clone(), headers, request, @@ -22876,11 +24389,14 @@ async fn send_known_worker_input( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let source = + authenticate_worker_mutation_source_owned(&api, &path.workspace_id, &headers).await?; let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; let request = worker_input_request_from_api(request)?; let idle_only = request.kind == WorkerInputKind::User; let result = worker.input(&context, request).await?; @@ -22912,11 +24428,14 @@ async fn cancel_known_worker( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let source = + authenticate_worker_mutation_source_owned(&api, &path.workspace_id, &headers).await?; let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; Ok(Json(worker_lifecycle_result_to_api( worker .cancel(&context, worker_lifecycle_request_from_api(request)) @@ -22931,11 +24450,14 @@ async fn stop_known_worker( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let source = + authenticate_worker_mutation_source_owned(&api, &path.workspace_id, &headers).await?; let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; Ok(Json(worker_lifecycle_result_to_api( worker .stop(&context, worker_lifecycle_request_from_api(request)) @@ -22950,7 +24472,8 @@ async fn restore_known_worker( request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let source = + authenticate_worker_mutation_source_owned(&api, &path.workspace_id, &headers).await?; let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; @@ -22969,41 +24492,46 @@ async fn scoped_list_worker_observation_sessions( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let sessions = api - .store - .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? - .into_iter() - .filter(|grant| { - grant - .permissions - .iter() - .any(|permission| permission == "observe") + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let sessions = api + .store + .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? + .into_iter() + .filter(|grant| { + grant + .permissions + .iter() + .any(|permission| permission == "observe") + }) + .filter_map(|grant| { + let worker = api.runtime.worker(&grant.subject).ok()?; + if matches!( + worker.state.as_str(), + "stopped" | "failed" | "rejected" | "disconnected" + ) { + return None; + } + Some(server_api::WorkerObservationSubject { + subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + display_name: worker.display_name, + relation: grant.relation, + status: worker.state, + }) + }) + .collect::>(); + Ok(Json(server_api::WorkerObservationSessionsResponse { + sessions, + })) }) - .filter_map(|grant| { - let worker = api.runtime.worker(&grant.subject).ok()?; - if matches!( - worker.state.as_str(), - "stopped" | "failed" | "rejected" | "disconnected" - ) { - return None; - } - Some(server_api::WorkerObservationSubject { - subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - display_name: worker.display_name, - relation: grant.relation, - status: worker.state, - }) - }) - .collect::>(); - Ok(Json(server_api::WorkerObservationSessionsResponse { - sessions, - })) + .await } async fn scoped_capture_worker_observation_session( @@ -23013,7 +24541,8 @@ async fn scoped_capture_worker_observation_session( Json(subject): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let source = + authenticate_worker_mutation_source_owned(&api, &path.workspace_id, &headers).await?; let server_api::WorkerObservationSubjectRef::RuntimeWorker { runtime_id, worker_id, @@ -23025,20 +24554,47 @@ async fn scoped_capture_worker_observation_session( }; let target = RuntimeWorkerRef::new(runtime_id, worker_id); let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let grant = authorize_known_worker_permission( - &api, - &path.workspace_id, - &controller, - &target, - "observe", - )?; + let grant_api = api.clone(); + let grant_controller = controller.clone(); + let grant_target = target.clone(); + let workspace_id = path.workspace_id.clone(); + let grant = api + .store + .db_executor() + .run(move || { + authorize_known_worker_permission( + &grant_api, + &workspace_id, + &grant_controller, + &grant_target, + "observe", + ) + }) + .await?; let lock = worker_control_lock(&api, &grant.grant_id); - let _guard = lock.lock().await; - authorize_known_worker_permission(&api, &path.workspace_id, &controller, &target, "observe")?; - let target_summary = api - .runtime - .worker(&target) - .map_err(|error| error.into_error())?; + let guard = lock.lock_owned().await; + let lookup_api = api.clone(); + let lookup_target = target.clone(); + let (_guard, target_summary) = api + .host_executors + .host()? + .run(move || { + // Retain the control guard during the actual re-fence and live lookup, + // even if the observation caller cancels its wait. + authorize_known_worker_permission( + &lookup_api, + &path.workspace_id, + &controller, + &lookup_target, + "observe", + )?; + let target_summary = lookup_api + .runtime + .worker(&lookup_target) + .map_err(|error| error.into_error())?; + Ok::<_, Error>((guard, target_summary)) + }) + .await?; if matches!( target_summary.state.as_str(), "stopped" | "failed" | "rejected" | "disconnected" @@ -23091,6 +24647,7 @@ async fn scoped_start_workspace_orchestrator( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { + api.host_executors.host()?.run(move || { #[cfg(test)] tests::seed_test_delivery_binding(&api); validate_workspace_scope(&api, &path.workspace_id)?; @@ -23217,6 +24774,8 @@ async fn scoped_start_workspace_orchestrator( drop(_guard); dispatch_orchestrator_assignment_attention(&api, &worker.worker); Ok(Json(workspace_orchestrator_response(&api, disposition))) + + }).await } fn worker_launch_worker_summary(worker: InternalWorkerSummary) -> WorkerLaunchWorkerSummary { @@ -23349,13 +24908,18 @@ async fn scoped_list_runtime_working_directories( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) + api.host_executors + .host()? + .run(move || { + let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) + }) + .await } async fn scoped_create_runtime_working_directory( @@ -23379,7 +24943,12 @@ async fn scoped_runtime_working_directory_detail( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) + api.host_executors + .host()? + .run(move || { + working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) + }) + .await } fn workdir_removal_source_actor( @@ -23465,19 +25034,25 @@ async fn cleanup_runtime_working_directory( ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - let registered_runtime = registered_workdir_runtime_id(&api, &path.working_directory_id)?; - if registered_runtime != path.runtime_id { - return Err(ApiError::from(Error::WorkspacePermissionDenied( - "Workdir does not belong to the requested Runtime".to_string(), - ))); - } - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) + api.host_executors + .host()? + .run(move || { + let registered_runtime = + registered_workdir_runtime_id(&api, &path.working_directory_id)?; + if registered_runtime != path.runtime_id { + return Err(ApiError::from(Error::WorkspacePermissionDenied( + "Workdir does not belong to the requested Runtime".to_string(), + ))); + } + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) + }) + .await } fn external_workdir_expires_at( @@ -23529,9 +25104,18 @@ fn schedule_external_workdir_expiry( } Err(error) => { tracing::error!(grant_id = %task_grant_id, error = ?error, "Backend External Workdir expiry cleanup requires retry"); + let stage_api = task_api.clone(); + let stage_grant_id = task_grant_id.clone(); let retryable = task_api .store - .get_external_workdir_grant(&task_api.config.workspace_id, &task_grant_id) + .db_executor() + .run(move || { + stage_api.store.get_external_workdir_grant( + &stage_api.config.workspace_id, + &stage_grant_id, + ) + }) + .await .map_or(true, |grant| grant.is_some()); if !retryable { break; @@ -23596,6 +25180,18 @@ async fn stop_external_workdir_provider( async fn fence_external_workdir_grant_expired( api: &WorkspaceApi, grant_id: &str, +) -> ApiResult { + let stage_api = api.clone(); + let grant_id = grant_id.to_string(); + api.store + .db_executor() + .run(move || fence_external_workdir_grant_expired_stage(&stage_api, &grant_id)) + .await +} + +fn fence_external_workdir_grant_expired_stage( + api: &WorkspaceApi, + grant_id: &str, ) -> ApiResult { loop { let mut grant = api @@ -23665,6 +25261,8 @@ async fn scoped_create_external_workdir_grant( Extension(actor): Extension, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { + api.store.db_executor().run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; let display_name = request.display_name.trim(); if display_name.is_empty() @@ -23734,6 +25332,8 @@ async fn scoped_create_external_workdir_grant( StatusCode::CREATED, Json(external_workdir_grant_response(&grant)?), )) + + }).await } fn external_workdir_grant_response( @@ -23758,16 +25358,21 @@ async fn scoped_external_workdir_provider_ws( AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ws: WebSocketUpgrade, ) -> ApiResult { - validate_workspace_scope(&api, &workspace_id)?; - let mut grant = api - .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir provider connection must use the grant creator's authenticated client".to_string(), - ).into()); - } + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); + let mut grant = api.store.db_executor().run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &stage_workspace_id)?; + let grant = stage_api.store + .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir provider connection must use the grant creator's authenticated client".to_string(), + ).into()); + } + Ok(grant) + }).await?; let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(&api, &grant_id).await?; @@ -23914,81 +25519,102 @@ async fn serve_external_workdir_provider( .await; return; } - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !api + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let registration_generation = registration.generation; + let registration_capabilities = registration.capabilities; + let activation = api .store - .activate_external_workdir_grant( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration.generation, - &now, - ) - .unwrap_or(false) - { - let _ = socket.close().await; - return; - } - if let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_connected", - &initial_grant.grant_id, - None, - "online", - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, registration.generation - ), - &now, - ) { - let _ = api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration.generation, - "offline", - &now, - ); - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = registration.generation, - %error, - "failed to audit External Workdir provider activation" - ); - let _ = socket.close().await; - return; - } - let expires_at = match external_workdir_expires_at(&initial_grant) { - Ok(value) => value, - Err(_) => { + .db_executor() + .run(move || -> Result<_> { + let api = stage_api; + let initial_grant = stage_grant; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !api + .store + .activate_external_workdir_grant( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration_generation, + &now, + ) + .unwrap_or(false) + { + return Ok(None); + } + if let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_connected", + &initial_grant.grant_id, + None, + "online", + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, registration_generation + ), + &now, + ) { + let _ = api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration_generation, + "offline", + &now, + ); + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = registration_generation, + %error, + "failed to audit External Workdir provider activation" + ); + return Ok(None); + } + let expires_at = match external_workdir_expires_at(&initial_grant) { + Ok(value) => value, + Err(_) => return Ok(None), + }; + let (sender, commands) = tokio::sync::mpsc::channel(32); + let connection = Arc::new(ExternalProviderConnection { + grant_id: initial_grant.grant_id.clone(), + workdir_id: initial_grant.workdir_id.clone(), + provider_instance_id: initial_grant.provider_instance_id.clone(), + generation: registration_generation, + expires_at, + capabilities: registration_capabilities, + admission: Arc::new(tokio::sync::Semaphore::new(16)), + shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + sender, + }); + // Publish with the accepted activation stage, never in its cancellable waiter. + if api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connected(connection.clone()) + .is_some() + { + return Ok(None); + } + Ok(Some((connection, commands))) + }) + .await; + let (connection, mut commands) = match activation { + Ok(Some(activated)) => activated, + Ok(None) => { + let _ = socket.close().await; + return; + } + Err(error) => { + tracing::error!(grant_id = %initial_grant.grant_id, %error, + "External Workdir provider activation stage failed"); let _ = socket.close().await; return; } }; - let (sender, mut commands) = tokio::sync::mpsc::channel(32); - let connection = Arc::new(ExternalProviderConnection { - grant_id: initial_grant.grant_id.clone(), - workdir_id: initial_grant.workdir_id.clone(), - provider_instance_id: initial_grant.provider_instance_id.clone(), - generation: registration.generation, - expires_at, - capabilities: registration.capabilities, - admission: Arc::new(tokio::sync::Semaphore::new(16)), - shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), - sender, - }); - if api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connected(connection.clone()) - .is_some() - { - let _ = socket.close().await; - return; - } let registered = ExternalWorkdirServerFrame::current(ExternalWorkdirServerMessage::Registered { generation: connection.generation, @@ -23996,19 +25622,36 @@ async fn serve_external_workdir_provider( max_in_flight_operations: 16, }); if !send_external_workdir_server_frame(&mut socket, ®istered).await { - let _ = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - if let Err(error) = api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &connection.provider_instance_id, - connection.generation, - "offline", - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let stage_connection = connection.clone(); + if let Err(error) = api + .store + .db_executor() + .run(move || { + stage_api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&stage_connection.grant_id, stage_connection.generation); + stage_api.store.update_external_workdir_grant_state( + &stage_grant.workspace_id, + &stage_grant.grant_id, + &stage_connection.provider_instance_id, + stage_connection.generation, + "offline", + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await + { + // Rejected admission must not leave a dead socket discoverable. + // The generation-bound in-memory fence is safe even for an unknown + // DB outcome; it neither guesses nor retries the durable mutation. + api.external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); tracing::error!( grant_id = %initial_grant.grant_id, generation = connection.generation, @@ -24158,55 +25801,81 @@ async fn serve_external_workdir_provider( } }; pending.clear(); - let _ = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - let state_updated = match api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &connection.provider_instance_id, - connection.generation, - terminal_status, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let stage_connection = connection.clone(); + let terminal_update = api + .store + .db_executor() + .run(move || -> Result<_> { + let api = stage_api; + let initial_grant = stage_grant; + let connection = stage_connection; + let _ = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + let state_updated = match api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &connection.provider_instance_id, + connection.generation, + terminal_status, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) { + Ok(updated) => updated, + Err(error) => { + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to persist External Workdir provider terminal state" + ); + false + } + }; + if state_updated + && let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_disconnected", + &initial_grant.grant_id, + None, + terminal_status, + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + { + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to audit External Workdir provider terminal state" + ); + } + Ok(state_updated) + }) + .await; + let state_updated = match terminal_update { Ok(updated) => updated, Err(error) => { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to persist External Workdir provider terminal state" - ); + api.external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + tracing::error!(grant_id = %initial_grant.grant_id, + generation = connection.generation, status = terminal_status, %error, + "External Workdir provider terminal stage failed"); false } }; - if state_updated - && let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_disconnected", - &initial_grant.grant_id, - None, - terminal_status, - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to audit External Workdir provider terminal state" - ); - } if state_updated && matches!(terminal_status, "expired" | "revoked") && let Err(error) = @@ -24226,17 +25895,27 @@ async fn scoped_get_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); let mut grant = api .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir grant inspection requires the grant creator".to_string(), - ) - .into()); - } + .db_executor() + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &stage_workspace_id)?; + let grant = stage_api + .store + .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir grant inspection requires the grant creator".to_string(), + ) + .into()); + } + Ok(grant) + }) + .await?; let expired = !matches!(grant.status.as_str(), "revoked" | "expired") && external_workdir_grant_is_expired(&grant)?; if expired { @@ -24263,44 +25942,57 @@ async fn scoped_revoke_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let mut grant = api + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); + let grant = api .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir revoke requires the grant creator".to_string(), - ) - .into()); - } - let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !matches!(grant.status.as_str(), "revoked" | "expired") { - api.store.update_external_workdir_grant_state( - &workspace_id, - &grant_id, - &grant.provider_instance_id, - grant.generation, - "revoked", - &updated_at, - )?; - grant.status = "revoked".to_string(); - grant.updated_at = updated_at; - api.store.record_external_workdir_audit( - &workspace_id, - "account", - &actor.account_id, - "external_workdir_grant_revoked", - &grant_id, - None, - "revoked", - &format!( - "workdir_id={} generation={}", - grant.workdir_id, grant.generation - ), - &grant.updated_at, - )?; - } + .db_executor() + .run(move || -> ApiResult<_> { + let api = stage_api; + let workspace_id = stage_workspace_id; + let grant_id = stage_grant_id; + validate_workspace_scope(&api, &workspace_id)?; + let mut grant = api + .store + .get_external_workdir_grant(&workspace_id, &grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir revoke requires the grant creator".to_string(), + ) + .into()); + } + let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !matches!(grant.status.as_str(), "revoked" | "expired") { + api.store.update_external_workdir_grant_state( + &workspace_id, + &grant_id, + &grant.provider_instance_id, + grant.generation, + "revoked", + &updated_at, + )?; + grant.status = "revoked".to_string(); + grant.updated_at = updated_at; + api.store.record_external_workdir_audit( + &workspace_id, + "account", + &actor.account_id, + "external_workdir_grant_revoked", + &grant_id, + None, + "revoked", + &format!( + "workdir_id={} generation={}", + grant.workdir_id, grant.generation + ), + &grant.updated_at, + )?; + } + Ok(grant) + }) + .await?; // Cleanup is deliberately retried for terminal grants. A prior revoke may // have committed its fence before a Runtime/session projection failure. let cleanup = cleanup_external_workdir_attachments(&api, &grant.workdir_id).await; @@ -24325,70 +26017,88 @@ async fn cleanup_external_workdir_attachments( async fn cleanup_external_workdir_attachments_with_lock( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<&RuntimeWorkerRef>, + already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, ) -> ApiResult<()> { - let workdir = api + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); + let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); + let grant_id = api .store - .get_workdir_registry(&api.config.workspace_id, workdir_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; - let grant_id = workdir.source.external_grant_id().ok_or_else(|| { - Error::InvalidInput("Workdir is not backed by an External grant".to_string()) - })?; - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "pending", - None, - &now, - )?; - - match release_external_workdir_attachments(api, workdir_id, already_locked_worker).await { - Ok(()) => { - let completed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "completed", + .db_executor() + .run(move || -> Result<_> { + let _session_guard = stage_guard; + let workdir = stage_api + .store + .get_workdir_registry(&stage_api.config.workspace_id, &stage_workdir_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; + let grant_id = workdir + .source + .external_grant_id() + .ok_or_else(|| { + Error::InvalidInput("Workdir is not backed by an External grant".to_string()) + })? + .to_string(); + stage_api.store.update_external_workdir_cleanup_state( + &stage_api.config.workspace_id, + &grant_id, + "pending", None, - &completed_at, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), )?; - api.store.record_external_workdir_audit( - &api.config.workspace_id, + Ok(grant_id) + }) + .await?; + + let release = + release_external_workdir_attachments(api, workdir_id, already_locked_worker).await; + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); + let succeeded = release.is_ok(); + let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); + api.store + .db_executor() + .run(move || -> Result<()> { + let _session_guard = stage_guard; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let (state, category, event, outcome, detail) = if succeeded { + ( + "completed", + None, + "external_workdir_cleanup_completed", + "completed", + format!("workdir_id={stage_workdir_id}"), + ) + } else { + ( + "retry_required", + Some("attachment_release_failed"), + "external_workdir_cleanup_attention_required", + "attention_required", + format!("workdir_id={stage_workdir_id} category=attachment_release_failed"), + ) + }; + stage_api.store.update_external_workdir_cleanup_state( + &stage_api.config.workspace_id, + &grant_id, + state, + category, + &now, + )?; + stage_api.store.record_external_workdir_audit( + &stage_api.config.workspace_id, "system", "workspace-backend", - "external_workdir_cleanup_completed", - grant_id, + event, + &grant_id, None, - "completed", - &format!("workdir_id={workdir_id}"), - &completed_at, + outcome, + &detail, + &now, )?; Ok(()) - } - Err(error) => { - let failed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "retry_required", - Some("attachment_release_failed"), - &failed_at, - )?; - api.store.record_external_workdir_audit( - &api.config.workspace_id, - "system", - "workspace-backend", - "external_workdir_cleanup_attention_required", - grant_id, - None, - "attention_required", - &format!("workdir_id={workdir_id} category=attachment_release_failed"), - &failed_at, - )?; - Err(error) - } - } + }) + .await?; + release } #[cfg(test)] @@ -24449,17 +26159,28 @@ async fn release_external_workdir_attachment_locked( api: &WorkspaceApi, workdir_id: &str, link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, ) -> ApiResult<()> { if link.unlinked_at.is_none() { close_current_worker_attachment_session_locked(api, &link.worker).await?; close_current_worker_command_sessions_for_alias_locked(api, &link.worker, &link.alias) .await?; - api.store.detach_worker_workdir( - &api.config.workspace_id, - &link.worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; + let stage_api = api.clone(); + let stage_worker = link.worker.clone(); + let stage_workdir_id = workdir_id.to_string(); + let stage_guard = session_guard.clone(); + api.store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + stage_api.store.detach_worker_workdir( + &stage_api.config.workspace_id, + &stage_worker, + Some(&stage_workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await?; #[cfg(test)] if take_external_workdir_cleanup_failure(workdir_id) { return Err(Error::RuntimeOperationFailed { @@ -24474,9 +26195,10 @@ async fn release_external_workdir_attachment_locked( // have committed detach before Runtime persistence or projection failed. #[cfg(test)] record_external_workdir_cleanup_reconciliation(workdir_id); - sync_runtime_worker_workdir_attachments(api, &link.worker)?; + sync_runtime_worker_workdir_attachments_locked(api, &link.worker, session_guard).await?; api.worker_projection - .refresh(&link.worker) + .refresh_async(&link.worker) + .await .map_err(ApiError::from)?; Ok(()) } @@ -24484,23 +26206,34 @@ async fn release_external_workdir_attachment_locked( async fn release_external_workdir_attachments( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<&RuntimeWorkerRef>, + already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, ) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); let Some(link) = api .store - .latest_workdir_worker_link(&api.config.workspace_id, workdir_id)? + .db_executor() + .run(move || { + stage_api + .store + .latest_workdir_worker_link(&stage_api.config.workspace_id, &stage_workdir_id) + }) + .await? else { return Ok(()); }; // The newest link is the only possible active attachment and remains the // durable reconciliation target if detach committed before Runtime update. // Missing/ambiguous Runtime state remains a retryable cleanup failure. - if already_locked_worker == Some(&link.worker) { - release_external_workdir_attachment_locked(api, workdir_id, &link).await?; + if let Some((worker, guard)) = + already_locked_worker.filter(|(worker, _)| *worker == &link.worker) + { + debug_assert_eq!(worker, &link.worker); + release_external_workdir_attachment_locked(api, workdir_id, &link, guard).await?; } else { let lock = current_worker_session_lock(api, &link.worker); - let _guard = lock.lock().await; - release_external_workdir_attachment_locked(api, workdir_id, &link).await?; + let guard = Arc::new(lock.lock_owned().await); + release_external_workdir_attachment_locked(api, workdir_id, &link, &guard).await?; } Ok(()) } @@ -24509,14 +26242,19 @@ async fn scoped_list_working_directories( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (items, diagnostics) = working_directory_summaries(&api)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (items, diagnostics) = working_directory_summaries(&api)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) + }) + .await } async fn scoped_create_working_directory( @@ -24531,40 +26269,47 @@ async fn scoped_working_directory_detail( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let record = api - .store - .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "working_directory_not_found".to_string(), - message: format!("Unknown Workdir `{}`", path.working_directory_id), - })?; - if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { - let grant = api - .store - .get_workspace_config_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| { - Error::WorkspacePermissionDenied("Workspace config grant unavailable".into()) - })?; - return Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: workspace_config::summary(&grant, &record), - diagnostics: vec![], - })); - } - if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { - working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) - } else { - let item = projected_workdir_summary_from_record(&api, &record)?; - Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: item.into(), - diagnostics: Vec::new(), - })) - } + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let record = api + .store + .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "working_directory_not_found".to_string(), + message: format!("Unknown Workdir `{}`", path.working_directory_id), + })?; + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Workspace config grant unavailable".into(), + ) + })?; + return Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: workspace_config::summary(&grant, &record), + diagnostics: vec![], + })); + } + if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { + working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) + } else { + let item = projected_workdir_summary_from_record(&api, &record)?; + Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: item.into(), + diagnostics: Vec::new(), + })) + } + }) + .await } async fn cleanup_working_directory( @@ -24574,13 +26319,18 @@ async fn cleanup_working_directory( request: WorkingDirectoryRemovalRequest, ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) + api.host_executors + .host()? + .run(move || { + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) + }) + .await } fn registered_workdir_runtime_id( @@ -24652,12 +26402,49 @@ fn finish_rejected_workdir_create_operation( Ok(failure_code) } -async fn create_workspace_working_directory( +async fn fail_workdir_create_stage( + api: &WorkspaceApi, + reserved: &WorkdirCreateOperationRecord, + request_fingerprint: &str, + failure: &str, + remove_registry: bool, +) -> Result { + let stage_api = api.clone(); + let reserved = reserved.clone(); + let request_fingerprint = request_fingerprint.to_string(); + let failure = failure.to_string(); + api.store + .db_executor() + .run(move || { + if remove_registry { + let _ = stage_api.store.delete_workdir_registry( + &reserved.workspace_id, + &reserved.working_directory_id, + ); + } + stage_api.config_store.finish_workdir_create_operation( + &reserved.workspace_id, + &reserved.operation_id, + &request_fingerprint, + false, + Some(&failure), + &now_registry_timestamp(), + ) + }) + .await +} + +fn prepare_workspace_workdir_create( api: &WorkspaceApi, workspace_id: &str, route_runtime_id: Option<&str>, mut request: BrowserWorkingDirectoryCreateRequest, -) -> ApiResult<(StatusCode, Json)> { +) -> ApiResult<( + WorkingDirectoryRequest, + String, + String, + WorkdirCreateOperationRecord, +)> { validate_workspace_scope(api, workspace_id)?; request.display_name = request .display_name @@ -24816,55 +26603,111 @@ async fn create_workspace_working_directory( })? }; - if reserved.state == "succeeded" { - return working_directory_detail_for_runtime( - api.clone(), - &reserved.resolved_runtime_id, - &reserved.working_directory_id, - ) - .map(|Json(response)| { - ( - StatusCode::OK, - Json(BrowserWorkingDirectoryCreateResponse { - workspace_id: response.workspace_id, - runtime_id: response - .runtime_id - .expect("Repository Workdir detail has Runtime"), - item: response.item, - diagnostics: response.diagnostics, - http_status: StatusCode::OK.as_u16(), - }), + Ok(( + working_directory_request, + operation_id, + request_fingerprint, + reserved, + )) +} + +async fn create_workspace_working_directory( + api: &WorkspaceApi, + workspace_id: &str, + route_runtime_id: Option<&str>, + request: BrowserWorkingDirectoryCreateRequest, +) -> ApiResult<(StatusCode, Json)> { + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.to_string(); + let route_runtime_id = route_runtime_id.map(str::to_string); + let (mut working_directory_request, operation_id, request_fingerprint, reserved) = api + .store + .db_executor() + .run(move || { + prepare_workspace_workdir_create( + &stage_api, + &stage_workspace_id, + route_runtime_id.as_deref(), + request, ) - }); + }) + .await?; + + if reserved.state == "succeeded" { + let stage_api = api.clone(); + return api + .host_executors + .host()? + .run(move || { + working_directory_detail_for_runtime( + stage_api, + &reserved.resolved_runtime_id, + &reserved.working_directory_id, + ) + .map(|Json(response)| { + ( + StatusCode::OK, + Json(BrowserWorkingDirectoryCreateResponse { + workspace_id: response.workspace_id, + runtime_id: response + .runtime_id + .expect("Repository Workdir detail has Runtime"), + item: response.item, + diagnostics: response.diagnostics, + http_status: StatusCode::OK.as_u16(), + }), + ) + }) + }) + .await; } let reserved = if reserved.state == "failed" { - api.config_store.begin_failed_workdir_create_retry( - workspace_id, - &operation_id, - &request_fingerprint, - &now_registry_timestamp(), - )? + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.to_string(); + let stage_operation_id = operation_id.clone(); + let stage_fingerprint = request_fingerprint.clone(); + api.store + .db_executor() + .run(move || { + stage_api.config_store.begin_failed_workdir_create_retry( + &stage_workspace_id, + &stage_operation_id, + &stage_fingerprint, + &now_registry_timestamp(), + ) + }) + .await? } else { reserved }; + let stage_api = api.clone(); + let stage_runtime_id = reserved.resolved_runtime_id.clone(); let runtime = match api - .runtime - .list_runtimes(usize::MAX) - .items - .into_iter() - .find(|runtime| runtime.runtime_id == reserved.resolved_runtime_id) + .host_executors + .host()? + .run(move || { + Ok::<_, Error>( + stage_api + .runtime + .list_runtimes(usize::MAX) + .items + .into_iter() + .find(|runtime| runtime.runtime_id == stage_runtime_id), + ) + }) + .await? { Some(runtime) => runtime, None => { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "runtime_unavailable", false, - Some("runtime_unavailable"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(ApiError::with_diagnostics( Error::UnknownRuntime(reserved.resolved_runtime_id), vec![RuntimeDiagnostic { @@ -24876,14 +26719,14 @@ async fn create_workspace_working_directory( } }; if runtime.kind == "remote_http" && runtime.status != "connected" { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "runtime_unavailable", false, - Some("runtime_unavailable"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: reserved.resolved_runtime_id, @@ -24898,38 +26741,52 @@ async fn create_workspace_working_directory( )); } - if let Err(error) = authorize_repository_materialization_operation( - api, - &reserved, - &request_fingerprint, - &mut working_directory_request, - ) { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let stage_fingerprint = request_fingerprint.clone(); + let (authorization, mut working_directory_request) = api + .store + .db_executor() + .run(move || { + let authorization = authorize_repository_materialization_operation( + &stage_api, + &stage_reserved, + &stage_fingerprint, + &mut working_directory_request, + ); + Ok::<_, Error>((authorization, working_directory_request)) + }) + .await?; + if let Err(error) = authorization { + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "working_directory_remote_repository_access_required", false, - Some("working_directory_remote_repository_access_required"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(error); } working_directory_request.backend_workdir_id = Some(reserved.working_directory_id.clone()); - let existing = match api.runtime.working_directory( - &reserved.resolved_runtime_id, - &reserved.working_directory_id, - ) { + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let existing = match api + .host_executors + .host()? + .run(move || { + Ok::<_, Error>(stage_api.runtime.working_directory( + &stage_reserved.resolved_runtime_id, + &stage_reserved.working_directory_id, + )) + }) + .await? + { Ok(existing) => existing, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, - &request_fingerprint, - false, - Some(&failure_code), - &now_registry_timestamp(), - )?; + fail_workdir_create_stage(api, &reserved, &request_fingerprint, &failure_code, false) + .await?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -24943,24 +26800,31 @@ async fn create_workspace_working_directory( let result = if existing.working_directory.is_some() { existing } else { + let stage_api = api.clone(); + let stage_runtime_id = reserved.resolved_runtime_id.clone(); match api - .runtime - .create_working_directory(&reserved.resolved_runtime_id, working_directory_request) + .host_executors + .host()? + .run(move || { + Ok::<_, Error>( + stage_api + .runtime + .create_working_directory(&stage_runtime_id, working_directory_request), + ) + }) + .await? { Ok(result) => result, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - let _ = api - .store - .delete_workdir_registry(workspace_id, &reserved.working_directory_id); - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, - false, - Some(&failure_code), - &now_registry_timestamp(), - )?; + &failure_code, + true, + ) + .await?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -24973,18 +26837,30 @@ async fn create_workspace_working_directory( } }; let Some(working_directory) = result.working_directory else { - let _ = api - .store - .delete_workdir_registry(workspace_id, &reserved.working_directory_id); let mut diagnostics = result.diagnostics; - let failure_code = finish_rejected_workdir_create_operation( - &api.config_store, - workspace_id, - &operation_id, - &request_fingerprint, - &diagnostics, - &now_registry_timestamp(), - )?; + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let stage_fingerprint = request_fingerprint.clone(); + let (failure_code, returned_diagnostics) = api + .store + .db_executor() + .run(move || -> Result<_> { + let _ = stage_api.store.delete_workdir_registry( + &stage_reserved.workspace_id, + &stage_reserved.working_directory_id, + ); + let failure_code = finish_rejected_workdir_create_operation( + &stage_api.config_store, + &stage_reserved.workspace_id, + &stage_reserved.operation_id, + &stage_fingerprint, + &diagnostics, + &now_registry_timestamp(), + )?; + Ok((failure_code, diagnostics)) + }) + .await?; + diagnostics = returned_diagnostics; if !diagnostics.iter().any(|diagnostic| { diagnostic.severity == HostDiagnosticSeverity::Error && diagnostic.code == failure_code }) { @@ -25006,21 +26882,29 @@ async fn create_workspace_working_directory( diagnostics, )); }; - let record = workdir_record_from_summary( - api, - &reserved.resolved_runtime_id, - &working_directory.summary, - ); - api.store.upsert_workdir_registry(&record)?; - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, - &request_fingerprint, - true, - None, - &now_registry_timestamp(), - )?; - let summary = projected_workdir_summary_from_record(api, &record)?; + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let summary = api + .store + .db_executor() + .run(move || -> ApiResult<_> { + let record = workdir_record_from_summary( + &stage_api, + &stage_reserved.resolved_runtime_id, + &working_directory.summary, + ); + stage_api.store.upsert_workdir_registry(&record)?; + stage_api.config_store.finish_workdir_create_operation( + &stage_reserved.workspace_id, + &stage_reserved.operation_id, + &request_fingerprint, + true, + None, + &now_registry_timestamp(), + )?; + Ok(projected_workdir_summary_from_record(&stage_api, &record)?) + }) + .await?; Ok(( StatusCode::CREATED, Json(BrowserWorkingDirectoryCreateResponse { @@ -25600,7 +27484,7 @@ async fn set_worker_retention( runtime_worker_id: String, pinned: bool, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &runtime_worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &runtime_worker_id).await?; Ok(Json( worker .set_pinned(&WorkerOperationContext::Backend, pinned) @@ -25864,7 +27748,13 @@ async fn execute_runtime_cleanup_with_context( } context.authorize_workspace(api).await?; - let plan = build_runtime_cleanup_plan(api, runtime_id)?; + let plan_api = api.clone(); + let plan_runtime = runtime_id.to_owned(); + let plan = api + .host_executors + .host()? + .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) + .await?; // Rebuild the actual candidate set immediately before executing. The digest // binds targets, dependencies and blockers; per-target authority is checked again below. if request.expected_plan_digest != plan.digest { @@ -25887,20 +27777,36 @@ async fn execute_runtime_cleanup_with_context( candidate.runtime_id.clone(), candidate.runtime_worker_id.clone(), ); - if let Some(claim) = api + let authority_api = api.clone(); + let target = worker.clone(); + let (claim, assignment) = api .store - .get_pending_ticket_assignment_operation_for_worker(&api.config.workspace_id, &worker)? - { + .db_executor() + .run(move || -> Result<_> { + Ok(( + authority_api + .store + .get_pending_ticket_assignment_operation_for_worker( + &authority_api.config.workspace_id, + &target, + )?, + authority_api + .store + .get_active_ticket_role_assignment_for_worker( + &authority_api.config.workspace_id, + &target, + )?, + )) + }) + .await?; + if let Some(claim) = claim { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_claim_pending", &format!("Worker has pending Ticket claim {}", claim.operation_id), )); } - if let Some(assignment) = api - .store - .get_active_ticket_role_assignment_for_worker(&api.config.workspace_id, &worker)? - { + if let Some(assignment) = assignment { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_assigned", @@ -25926,7 +27832,8 @@ async fn execute_runtime_cleanup_with_context( )); } parse_runtime_worker_id_for_registry(&candidate.runtime_worker_id)?; - WorkspaceWorker::resolve(api, &worker.runtime_id, &worker.worker_id)? + WorkspaceWorker::resolve_async(api, &worker.runtime_id, &worker.worker_id) + .await? .delete_from_plan(context, candidate) .await?; api.workdir_session_locks @@ -25965,12 +27872,21 @@ async fn execute_runtime_cleanup_with_context( }); } CleanupTargetKind::WorkdirCleanCleanup | CleanupTargetKind::WorkdirRecordDelete => { - let removal = execute_workdir_removal( - api, - candidate.workdir_id.as_str(), - &format!("runtime-cleanup:{runtime_id}"), - &format!("cleanup target {}", candidate.target_id), - )?; + let removal_api = api.clone(); + let removal_candidate = candidate.clone(); + let requester = format!("runtime-cleanup:{runtime_id}"); + let removal = api + .host_executors + .host()? + .run(move || { + execute_workdir_removal( + &removal_api, + removal_candidate.workdir_id.as_str(), + &requester, + &format!("cleanup target {}", removal_candidate.target_id), + ) + }) + .await?; let (status, message) = match removal.disposition { WorkingDirectoryRemovalDisposition::Removed => ( "deleted", @@ -26016,7 +27932,13 @@ async fn execute_runtime_cleanup_with_context( )); } - let plan_after = build_runtime_cleanup_plan(api, runtime_id)?; + let plan_api = api.clone(); + let plan_runtime = runtime_id.to_owned(); + let plan_after = api + .host_executors + .host()? + .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) + .await?; let executed_at = now_registry_timestamp(); Ok(RuntimeCleanupExecutionResponse { workspace_id: api.config.workspace_id.clone(), @@ -26084,30 +28006,44 @@ async fn scoped_update_remote_runtime( .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) - .unwrap_or(&path.runtime_id); - let binding = api - .store - .update_workspace_runtime_binding_metadata( - &path.workspace_id, - &path.runtime_id, - display_name, - endpoint.as_str().trim_end_matches('/'), - &Utc::now().to_rfc3339(), - ) - .await?; - api.runtime_binding_expectations - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - (path.workspace_id.clone(), path.runtime_id.clone()), - binding.clone(), - ); - api.runtime_subscription_broker - .unregister_runtime(&path.runtime_id); - api.register_workspace_runtime_binding(binding)?; - Ok(Json( - workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id).await?, - )) + .unwrap_or(&path.runtime_id) + .to_owned(); + update_remote_runtime_metadata_owned( + api, + path.workspace_id, + path.runtime_id, + display_name, + endpoint.as_str().trim_end_matches('/').to_owned(), + ) + .await +} + +// Post-auth mutation boundary: cancellation cannot split SQL completion from +// publication or discard the accepted workflow's request/resource authority. +async fn update_remote_runtime_metadata_owned( + api: WorkspaceApi, + workspace_id: String, + runtime_id: String, + display_name: String, + endpoint: String, +) -> ApiResult> { + run_host_workflow(api.host_executors.host()?, async move { + let binding = api + .store + .update_workspace_runtime_binding_metadata( + &workspace_id, + &runtime_id, + &display_name, + &endpoint, + &Utc::now().to_rfc3339(), + ) + .await?; + publish_workspace_runtime_binding(&api, binding)?; + Ok(Json( + workspace_runtime_detail_on_host(&api, &workspace_id, &runtime_id).await?, + )) + }) + .await } async fn scoped_reveal_runtime_trust_key( @@ -26289,7 +28225,9 @@ async fn scoped_restore_runtime_worker_with_context( request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let resolved = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; let _authorization = resolved.authorize_operation(&context, "restore").await?; let runtime_id = resolved.identity().runtime_id.clone(); let worker_id = resolved.identity().worker_id.clone(); @@ -26322,9 +28260,14 @@ async fn scoped_runtime_cleanup_plan( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; - Ok(Json(plan)) + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; + Ok(Json(plan)) + }) + .await } #[cfg(test)] @@ -26369,56 +28312,65 @@ async fn scoped_create_attachment_upload_grant( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - resolve_workspace_worker_reference(&api, &path.worker.runtime_id, &path.worker.worker_id)?; - let now = attachment_upload_now_ms()?; - let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); - let upload_id = match query.upload_id { - Some(upload_id) => Uuid::parse_str(&upload_id) - .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? - .to_string(), - None => Uuid::now_v7().to_string(), - }; - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - grants.retain(|_, grant| grant.expires_at_ms >= now); - if let Some(existing) = grants.get(&upload_id) { - if existing.workspace_id == path.workspace_id - && existing.runtime_id == path.worker.runtime_id - && existing.worker_id == path.worker.worker_id - && existing.account_id == actor.account_id - && existing.file_name == query.file_name - && existing.media_type == query.media_type - { - return Ok(Json(AttachmentUploadGrantResponse { + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + resolve_workspace_worker_reference( + &api, + &path.worker.runtime_id, + &path.worker.worker_id, + )?; + let now = attachment_upload_now_ms()?; + let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); + let upload_id = match query.upload_id { + Some(upload_id) => Uuid::parse_str(&upload_id) + .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? + .to_string(), + None => Uuid::now_v7().to_string(), + }; + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + grants.retain(|_, grant| grant.expires_at_ms >= now); + if let Some(existing) = grants.get(&upload_id) { + if existing.workspace_id == path.workspace_id + && existing.runtime_id == path.worker.runtime_id + && existing.worker_id == path.worker.worker_id + && existing.account_id == actor.account_id + && existing.file_name == query.file_name + && existing.media_type == query.media_type + { + return Ok(Json(AttachmentUploadGrantResponse { + upload_id, + expires_at_ms: existing.expires_at_ms, + })); + } + return Err(Error::RepositoryConflict( + "attachment upload id was reused with different intent".into(), + ) + .into()); + } + grants.insert( + upload_id.clone(), + AttachmentUploadGrant { + workspace_id: path.workspace_id, + runtime_id: path.worker.runtime_id, + worker_id: path.worker.worker_id, + account_id: actor.account_id, + file_name: query.file_name, + media_type: query.media_type, + expires_at_ms, + state: AttachmentUploadGrantState::Pending, + }, + ); + Ok(Json(AttachmentUploadGrantResponse { upload_id, - expires_at_ms: existing.expires_at_ms, - })); - } - return Err(Error::RepositoryConflict( - "attachment upload id was reused with different intent".into(), - ) - .into()); - } - grants.insert( - upload_id.clone(), - AttachmentUploadGrant { - workspace_id: path.workspace_id, - runtime_id: path.worker.runtime_id, - worker_id: path.worker.worker_id, - account_id: actor.account_id, - file_name: query.file_name, - media_type: query.media_type, - expires_at_ms, - state: AttachmentUploadGrantState::Pending, - }, - ); - Ok(Json(AttachmentUploadGrantResponse { - upload_id, - expires_at_ms, - })) + expires_at_ms, + })) + }) + .await } async fn upload_runtime_worker_file_contract( @@ -26427,75 +28379,84 @@ async fn upload_runtime_worker_file_contract( path: ScopedAttachmentUploadPath, body: &[u8], ) -> ApiResult { - validate_workspace_scope(&api, &path.workspace_id)?; - let body_sha256 = attachment_body_sha256(body); - let grant = { - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - let grant = grants - .get_mut(&path.upload_id) - .ok_or_else(|| Error::InvalidInput("attachment upload grant is unknown".into()))?; - if let Some(file) = grant - .claim( - &path.workspace_id, - &path.runtime_id, - &path.worker_id, - &actor.account_id, - attachment_upload_now_ms()?, - &body_sha256, - ) - .map_err(ApiError::from)? - { - return Ok(WorkerFileUploadResponse { file }); - } - grant.clone() - }; + let body = body.to_vec(); + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let body_sha256 = attachment_body_sha256(&body); + let grant = { + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + let grant = grants.get_mut(&path.upload_id).ok_or_else(|| { + Error::InvalidInput("attachment upload grant is unknown".into()) + })?; + if let Some(file) = grant + .claim( + &path.workspace_id, + &path.runtime_id, + &path.worker_id, + &actor.account_id, + attachment_upload_now_ms()?, + &body_sha256, + ) + .map_err(ApiError::from)? + { + return Ok(WorkerFileUploadResponse { file }); + } + grant.clone() + }; - let worker = resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; - let upload_context = worker_runtime::UploadedFileUploadContext { - upload_id: path.upload_id.clone(), - principal_id: grant.account_id.clone(), - workspace_id: grant.workspace_id.clone(), - runtime_id: grant.runtime_id.clone(), - worker_id: grant.worker_id.clone(), - }; - let uploaded = api.runtime.upload_worker_file( - &worker, - &grant.file_name, - &grant.media_type, - body, - Some(&upload_context), - ); - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - let current = grants - .get_mut(&path.upload_id) - .ok_or_else(|| Error::RepositoryConflict("attachment upload grant disappeared".into()))?; - if matches!(current.state, AttachmentUploadGrantState::Cancelled) { - let uploaded_file = uploaded.ok(); - grants.remove(&path.upload_id); - drop(grants); - if let Some(file) = uploaded_file { - api.runtime - .delete_worker_uploaded_file(&worker, &file.artifact_id) - .map_err(|error| error.into_error())?; - } - return Err(Error::RepositoryConflict("attachment upload was cancelled".into()).into()); - } - match uploaded { - Ok(file) => { - current.state = AttachmentUploadGrantState::Completed(file.clone()); - Ok(WorkerFileUploadResponse { file }) - } - Err(error) => { - current.state = AttachmentUploadGrantState::Pending; - Err(error.into_error().into()) - } - } + let worker = + resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; + let upload_context = worker_runtime::UploadedFileUploadContext { + upload_id: path.upload_id.clone(), + principal_id: grant.account_id.clone(), + workspace_id: grant.workspace_id.clone(), + runtime_id: grant.runtime_id.clone(), + worker_id: grant.worker_id.clone(), + }; + let uploaded = api.runtime.upload_worker_file( + &worker, + &grant.file_name, + &grant.media_type, + &body, + Some(&upload_context), + ); + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + let current = grants.get_mut(&path.upload_id).ok_or_else(|| { + Error::RepositoryConflict("attachment upload grant disappeared".into()) + })?; + if matches!(current.state, AttachmentUploadGrantState::Cancelled) { + let uploaded_file = uploaded.ok(); + grants.remove(&path.upload_id); + drop(grants); + if let Some(file) = uploaded_file { + api.runtime + .delete_worker_uploaded_file(&worker, &file.artifact_id) + .map_err(|error| error.into_error())?; + } + return Err( + Error::RepositoryConflict("attachment upload was cancelled".into()).into(), + ); + } + match uploaded { + Ok(file) => { + current.state = AttachmentUploadGrantState::Completed(file.clone()); + Ok(WorkerFileUploadResponse { file }) + } + Err(error) => { + current.state = AttachmentUploadGrantState::Pending; + Err(error.into_error().into()) + } + } + }) + .await } async fn scoped_cancel_attachment_upload( @@ -26503,49 +28464,55 @@ async fn scoped_cancel_attachment_upload( Extension(actor): Extension, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let completed = { - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - let grant = grants - .get_mut(&path.upload_id) - .ok_or_else(|| Error::InvalidInput("attachment upload grant is unknown".into()))?; - if grant.workspace_id != path.workspace_id - || grant.runtime_id != path.runtime_id - || grant.worker_id != path.worker_id - || grant.account_id != actor.account_id - { - return Err(Error::WorkspacePermissionDenied( - "attachment upload grant scope mismatch".into(), - ) - .into()); - } - match &grant.state { - AttachmentUploadGrantState::Pending => { - grants.remove(&path.upload_id); - None + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let completed = { + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + let grant = grants.get_mut(&path.upload_id).ok_or_else(|| { + Error::InvalidInput("attachment upload grant is unknown".into()) + })?; + if grant.workspace_id != path.workspace_id + || grant.runtime_id != path.runtime_id + || grant.worker_id != path.worker_id + || grant.account_id != actor.account_id + { + return Err(Error::WorkspacePermissionDenied( + "attachment upload grant scope mismatch".into(), + ) + .into()); + } + match &grant.state { + AttachmentUploadGrantState::Pending => { + grants.remove(&path.upload_id); + None + } + AttachmentUploadGrantState::Uploading => { + grant.state = AttachmentUploadGrantState::Cancelled; + None + } + AttachmentUploadGrantState::Cancelled => None, + AttachmentUploadGrantState::Completed(file) => { + let file = file.clone(); + grants.remove(&path.upload_id); + Some(file) + } + } + }; + if let Some(file) = completed { + let worker = + resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &file.artifact_id) + .map_err(|error| error.into_error())?; } - AttachmentUploadGrantState::Uploading => { - grant.state = AttachmentUploadGrantState::Cancelled; - None - } - AttachmentUploadGrantState::Cancelled => None, - AttachmentUploadGrantState::Completed(file) => { - let file = file.clone(); - grants.remove(&path.upload_id); - Some(file) - } - } - }; - if let Some(file) = completed { - let worker = resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &file.artifact_id) - .map_err(|error| error.into_error())?; - } - Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) + Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) + }) + .await } async fn scoped_delete_runtime_worker_uploaded_file( @@ -26622,8 +28589,13 @@ async fn post_auth_bootstrap_user( State(api): State, Json(request): Json, ) -> ApiResult> { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - Ok(Json(AuthUserResponse { user })) + api.store + .db_executor() + .run(move || { + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + Ok(Json(AuthUserResponse { user })) + }) + .await } async fn post_passkey_registration_options( @@ -26631,48 +28603,53 @@ async fn post_passkey_registration_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let exclude_credentials = passkeys_for_user(&api, &user.user_id)? - .into_iter() - .map(|passkey| passkey.cred_id().clone()) - .collect(); - let user_unique_id = Uuid::now_v7(); - let (public_key, state) = webauthn - .start_passkey_registration( - user_unique_id, - &user.handle, - &user.display_name, - Some(exclude_credentials), - ) - .map_err(|error| auth_error("webauthn_registration_options_failed", &error.to_string()))?; - let challenge_id = new_id("webauthn-registration"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_registration".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some( - serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?, - ), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyRegistrationOptionsResponse { - challenge_id, - public_key, - })) + api.store + .db_executor() + .run(move || { + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let exclude_credentials = passkeys_for_user(&api, &user.user_id)? + .into_iter() + .map(|passkey| passkey.cred_id().clone()) + .collect(); + let user_unique_id = Uuid::now_v7(); + let (public_key, state) = webauthn + .start_passkey_registration( + user_unique_id, + &user.handle, + &user.display_name, + Some(exclude_credentials), + ) + .map_err(|error| { + auth_error("webauthn_registration_options_failed", &error.to_string()) + })?; + let challenge_id = new_id("webauthn-registration"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_registration".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some(serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyRegistrationOptionsResponse { + challenge_id, + public_key, + })) + }) + .await } fn complete_passkey_registration( @@ -26749,50 +28726,53 @@ async fn post_passkey_login_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - let user = match request.handle.as_deref() { - Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, - None => api.store.any_user()?, - } - .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; - let passkeys = passkeys_for_user(&api, &user.user_id)?; - if passkeys.is_empty() { - return Err(auth_error( - "passkey_not_registered", - "user has no registered passkey credentials", - ) - .into()); - } - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let (public_key, state) = webauthn - .start_passkey_authentication(&passkeys) - .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; - let challenge_id = new_id("webauthn-login"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_login".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some( - serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?, - ), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyLoginOptionsResponse { - challenge_id, - public_key, - })) + api.store + .db_executor() + .run(move || { + let user = match request.handle.as_deref() { + Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, + None => api.store.any_user()?, + } + .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; + let passkeys = passkeys_for_user(&api, &user.user_id)?; + if passkeys.is_empty() { + return Err(auth_error( + "passkey_not_registered", + "user has no registered passkey credentials", + ) + .into()); + } + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let (public_key, state) = webauthn + .start_passkey_authentication(&passkeys) + .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; + let challenge_id = new_id("webauthn-login"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_login".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some(serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyLoginOptionsResponse { + challenge_id, + public_key, + })) + }) + .await } fn complete_passkey_login( @@ -26910,48 +28890,53 @@ async fn post_device_login_start( State(api): State, Json(request): Json, ) -> ApiResult> { - const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; + api.store + .db_executor() + .run(move || { + const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; - let auth = auth_public_config(&api.config); - let verification_uri = format!( - "{}/login/device", - auth.public_base_url.trim_end_matches('/') - ); - for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { - let device_code = mint_secret("yoi_device"); - let user_code = new_user_code(); - let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); - let flow = DeviceLoginFlowRecord { - device_code: device_code.clone(), - user_code: user_code.clone(), - verification_uri: verification_uri.clone(), - client_name: request.client_name.clone(), - user_id: None, - api_token_id: None, - issued_access_token: None, - created_at: crate::auth::now_rfc3339(), - expires_at: rfc3339_after(Duration::minutes(10)), - approved_at: None, - consumed_at: None, - }; - if !api.store.try_create_device_login_flow(&flow)? { - continue; - } - return Ok(Json(DeviceLoginStartResponse { - device_code, - user_code, - verification_uri, - verification_uri_complete, - expires_in: 600, - interval: 5, - })); - } + let auth = auth_public_config(&api.config); + let verification_uri = format!( + "{}/login/device", + auth.public_base_url.trim_end_matches('/') + ); + for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { + let device_code = mint_secret("yoi_device"); + let user_code = new_user_code(); + let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); + let flow = DeviceLoginFlowRecord { + device_code: device_code.clone(), + user_code: user_code.clone(), + verification_uri: verification_uri.clone(), + client_name: request.client_name.clone(), + user_id: None, + api_token_id: None, + issued_access_token: None, + created_at: crate::auth::now_rfc3339(), + expires_at: rfc3339_after(Duration::minutes(10)), + approved_at: None, + consumed_at: None, + }; + if !api.store.try_create_device_login_flow(&flow)? { + continue; + } + return Ok(Json(DeviceLoginStartResponse { + device_code, + user_code, + verification_uri, + verification_uri_complete, + expires_in: 600, + interval: 5, + })); + } - Err(auth_error( - "device_login_code_allocation_failed", - "could not allocate a unique device login user code", - ) - .into()) + Err(auth_error( + "device_login_code_allocation_failed", + "could not allocate a unique device login user code", + ) + .into()) + }) + .await } fn approve_device_login( @@ -27017,41 +29002,48 @@ async fn post_device_login_poll( State(api): State, Json(request): Json, ) -> ApiResult> { - let Some(flow) = api - .store - .get_device_login_flow_by_device_code(&request.device_code)? - else { - return Err(auth_error("unknown_device_login", "device login flow does not exist").into()); - }; - if is_expired(&flow.expires_at) { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Expired, - access_token: None, - token_type: None, - })); - } - if flow.approved_at.is_none() { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Pending, - access_token: None, - token_type: None, - })); - } - let Some(consumed) = api - .store - .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? - else { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Consumed, - access_token: None, - token_type: None, - })); - }; - Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Approved, - access_token: consumed.issued_access_token, - token_type: Some(DeviceAccessTokenType::Bearer), - })) + api.store + .db_executor() + .run(move || { + let Some(flow) = api + .store + .get_device_login_flow_by_device_code(&request.device_code)? + else { + return Err( + auth_error("unknown_device_login", "device login flow does not exist").into(), + ); + }; + if is_expired(&flow.expires_at) { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Expired, + access_token: None, + token_type: None, + })); + } + if flow.approved_at.is_none() { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Pending, + access_token: None, + token_type: None, + })); + } + let Some(consumed) = api + .store + .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? + else { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Consumed, + access_token: None, + token_type: None, + })); + }; + Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Approved, + access_token: consumed.issued_access_token, + token_type: Some(DeviceAccessTokenType::Bearer), + })) + }) + .await } fn logout_browser_session( @@ -27245,7 +29237,7 @@ async fn resolve_actor( headers: &HeaderMap, ) -> ApiResult> { let cookie_name = auth_public_config(&api.config).cookie_name; - Ok(resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await?) + Ok(resolve_request_actor(api.store.clone(), headers, &cookie_name).await?) } async fn require_actor(api: &ServerAuthApi, headers: &HeaderMap) -> ApiResult { @@ -27337,103 +29329,130 @@ async fn list_tickets( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(30).clamp(1, 100); - let states = query - .states - .as_deref() - .map(|states| { - states - .split(',') - .filter(|state| !state.is_empty()) - .map(str::to_string) - .collect::>() + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(30).clamp(1, 100); + let states = query + .states + .as_deref() + .map(|states| { + states + .split(',') + .filter(|state| !state.is_empty()) + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default(); + let crate::records::TicketSummaryPage { + items, + page, + invalid_records, + record_authority, + } = api + .authority + .list_ticket_page(crate::records::TicketListProjectionRequest { + states, + limit: Some(limit), + cursor: query.cursor, + })?; + Ok(Json(crate::records::TicketListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + page, + invalid_records, + record_authority, + })) }) - .unwrap_or_default(); - let crate::records::TicketSummaryPage { - items, - page, - invalid_records, - record_authority, - } = api - .authority - .list_ticket_page(crate::records::TicketListProjectionRequest { - states, - limit: Some(limit), - cursor: query.cursor, - })?; - Ok(Json(crate::records::TicketListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - page, - invalid_records, - record_authority, - })) + .await } async fn get_ticket( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - Ok(Json(api.authority.ticket(&id)?)) + api.host_executors + .host()? + .run(move || Ok(Json(api.authority.ticket(&id)?))) + .await } async fn list_objectives( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(api.config.max_records).min(1000); - let ProjectRecordList { - items, - invalid_records, - record_authority, - } = api.authority.list_objectives(limit)?; - Ok(Json(server_api::ObjectiveListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - invalid_records, - record_authority, - })) + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(api.config.max_records).min(1000); + let ProjectRecordList { + items, + invalid_records, + record_authority, + } = api.authority.list_objectives(limit)?; + Ok(Json(server_api::ObjectiveListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + invalid_records, + record_authority, + })) + }) + .await } async fn get_objective( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - Ok(Json(api.authority.objective(&id)?)) + api.store + .db_executor() + .run(move || Ok(Json(api.authority.objective(&id)?))) + .await } async fn list_repositories( State(api): State, ) -> ApiResult> { - let RepositoryListProjection { items, diagnostics } = api.live_repository_reader()?.list(); - Ok(Json(RepositoryListResponse { - workspace_id: api.config.workspace_id, - items, - source: "workspace-control-plane".to_string(), - diagnostics, - })) + api.host_executors + .host()? + .run(move || { + let RepositoryListProjection { items, diagnostics } = + api.live_repository_reader()?.list(); + Ok(Json(RepositoryListResponse { + workspace_id: api.config.workspace_id, + items, + source: "workspace-control-plane".to_string(), + diagnostics, + })) + }) + .await } async fn repository_detail( State(api): State, AxumPath(repository_key): AxumPath, ) -> ApiResult> { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let item = repository_lookup( - api.live_repository_reader()? - .summary_by_key(&repository_key), - )?; - Ok(Json(RepositoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - item, - source: "workspace-control-plane".to_string(), - })) + api.host_executors + .host()? + .run(move || { + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let item = repository_lookup( + api.live_repository_reader()? + .summary_by_key(&repository_key), + )?; + Ok(Json(RepositoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + item, + source: "workspace-control-plane".to_string(), + })) + }) + .await } async fn repository_log( @@ -27441,63 +29460,150 @@ async fn repository_log( AxumPath(repository_key): AxumPath, Query(query): Query, ) -> ApiResult> { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let RepositoryLogRead { - repository_key, - default_selector, - limit, - commits, - diagnostics, - } = repository_lookup( - api.live_repository_reader()? - .recent_log(&repository_key, query.limit), - )?; - Ok(Json(RepositoryLogResponse { - workspace_id: api.config.workspace_id, - repository_key, - default_selector, - limit, - items: commits, - diagnostics, - })) + api.host_executors + .host()? + .run(move || { + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let RepositoryLogRead { + repository_key, + default_selector, + limit, + commits, + diagnostics, + } = repository_lookup( + api.live_repository_reader()? + .recent_log(&repository_key, query.limit), + )?; + Ok(Json(RepositoryLogResponse { + workspace_id: api.config.workspace_id, + repository_key, + default_selector, + limit, + items: commits, + diagnostics, + })) + }) + .await } async fn list_hosts( State(api): State, ) -> ApiResult>> { - let limit = api.config.max_records.min(200); - let runtime_hosts = api.runtime.list_hosts(limit); - Ok(Json(InternalRuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtime_hosts.items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_hosts.diagnostics, - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtime_hosts = api.runtime.list_hosts(limit); + Ok(Json(InternalRuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtime_hosts.items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_hosts.diagnostics, + })) + }) + .await } async fn list_runtimes( State(api): State, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let runtimes = api.runtime.list_runtimes(limit); - Ok(Json(server_api::RuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtimes.items.into_iter().map(Into::into).collect(), - source: "worker_runtime_registry".to_string(), - diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtimes = api.runtime.list_runtimes(limit); + Ok(Json(server_api::RuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtimes.items.into_iter().map(Into::into).collect(), + source: "worker_runtime_registry".to_string(), + diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), + })) + }) + .await } async fn list_workers( State(api): State, ) -> ApiResult> { - workers_response(api).map(Json) + api.store + .db_executor() + .run(move || workers_response(api).map(Json)) + .await +} + +async fn require_active_runtime_registration_identity(api: &WorkspaceApi) -> ApiResult<()> { + let identities = api.signing_identities.clone(); + let workspace_id = api.config.workspace_id.clone(); + api.store + .db_executor() + .run(move || { + let identity = identities.get_validated(&workspace_id)?; + if identity.state != "active" { + return Err(settings_bad_request( + "workspace_signing_identity_unavailable", + "an active Workspace signing identity is required before registering a Runtime", + )); + } + Ok(()) + }) + .await +} + +async fn scan_runtime_registration_workers(api: &WorkspaceApi, runtime_id: &str) -> ApiResult<()> { + let runtime = api.runtime.clone(); + let runtime_id = runtime_id.to_owned(); + let limit = api.config.max_records.min(200); + api.host_executors.host()?.run(move || { + match runtime.list_workers_for_runtime(&runtime_id, limit) { + Ok(workers) if !workers.items.is_empty() => Err(Error::RuntimeBindingConflict(format!( + "Runtime `{runtime_id}` still has {} active worker(s) and cannot replace its connection", + workers.items.len() + )).into()), + Ok(_) | Err(RuntimeRegistryError::UnknownRuntime(_)) => Ok(()), + Err(error) => Err(error.into_error().into()), + } + }).await +} + +fn publish_workspace_runtime_binding( + api: &WorkspaceApi, + binding: WorkspaceRuntimeBinding, +) -> Result<()> { + api.runtime_binding_expectations + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + (binding.workspace_id.clone(), binding.runtime_id.clone()), + binding.clone(), + ); + api.runtime_subscription_broker + .unregister_runtime(&binding.runtime_id); + api.register_workspace_runtime_binding(binding) +} + +async fn require_current_runtime_test_binding( + api: &WorkspaceApi, + binding: WorkspaceRuntimeBinding, +) -> ApiResult { + let store = api.store.clone(); + api.store + .db_executor() + .run(move || { + if !store.workspace_runtime_binding_matches(&binding)? { + return Err(Error::RuntimeBindingConflict( + "Runtime connection changed while the connection test was running".to_string(), + ) + .into()); + } + Ok(binding) + }) + .await } async fn create_remote_runtime( @@ -27520,33 +29626,13 @@ async fn create_remote_runtime( "the embedded Runtime is built in and cannot be managed as a remote Runtime", )); } - let identity = api - .signing_identities - .get_validated(&api.config.workspace_id)?; - if identity.state != "active" { - return Err(settings_bad_request( - "workspace_signing_identity_unavailable", - "an active Workspace signing identity is required before registering a Runtime", - )); - } + require_active_runtime_registration_identity(&api).await?; // Protect observed active Workers without removing the current connection. // This scan is not proof of idleness: an unavailable/auth-failing endpoint // must remain repairable by an owner using binding-ID CAS below. Replacement // neither stops nor deletes Workers; Runtime removal has separate preflight. // Rejected key/CAS mutations must leave the current registration usable. - match api - .runtime - .list_workers_for_runtime(&runtime_id, api.config.max_records.min(200)) - { - Ok(workers) if !workers.items.is_empty() => { - return Err(Error::RuntimeBindingConflict(format!( - "Runtime `{runtime_id}` still has {} active worker(s) and cannot replace its connection", - workers.items.len() - )).into()); - } - Ok(_) | Err(RuntimeRegistryError::UnknownRuntime(_)) => {} - Err(error) => return Err(error.into_error().into()), - } + scan_runtime_registration_workers(&api, &runtime_id).await?; let now = Utc::now().to_rfc3339(); let record = WorkspaceRuntimeBinding { workspace_id: api.config.workspace_id.clone(), @@ -27567,30 +29653,32 @@ async fn create_remote_runtime( updated_at: now, revoked_at: None, }; - let (mutation, binding) = api - .store - .put_workspace_runtime_binding_key( - record, - request.expected_binding_id.as_deref(), - &actor.account_id, + let mutation_api = api.clone(); + // One accepted owner keeps SQL commit, local registration and its response + // observation together even when the HTTP waiter disappears. Resource reads + // call the on-Host helper directly, never re-admit to the occupied Host lane. + let (mutation, resource) = run_host_workflow(api.host_executors.host()?, async move { + let (mutation, binding) = mutation_api + .store + .put_workspace_runtime_binding_key( + record, + request.expected_binding_id.as_deref(), + &actor.account_id, + ) + .await?; + publish_workspace_runtime_binding(&mutation_api, binding)?; + let resource = workspace_runtime_resources_response_on_host( + &mutation_api, + mutation_api.workspace_id(), ) - .await?; - api.runtime_binding_expectations - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - (api.config.workspace_id.clone(), runtime_id.clone()), - binding.clone(), - ); - api.runtime_subscription_broker - .unregister_runtime(&runtime_id); - api.register_workspace_runtime_binding(binding)?; - let resource = workspace_runtime_resources_response(&api, &api.config.workspace_id) .await? .items .into_iter() .find(|resource| resource.runtime.runtime_id == runtime_id) .ok_or_else(|| Error::UnknownRuntime(runtime_id.clone()))?; + Ok::<_, ApiError>((mutation, resource)) + }) + .await?; let status = if mutation == WorkspaceRuntimeBindingMutation::Created { StatusCode::CREATED } else { @@ -27669,6 +29757,19 @@ fn runtime_removal_config_guard( async fn execute_runtime_removal( api: &WorkspaceApi, operation: RuntimeRemovalOperation, +) -> ApiResult { + let api = api.clone(); + // Retain registration/commit recovery as one owned lifecycle workflow; + // SQL transaction boundaries and commit-before-Runtime order are unchanged. + run_host_workflow(api.host_executors.host()?, async move { + execute_runtime_removal_on_host(&api, operation).await + }) + .await +} + +async fn execute_runtime_removal_on_host( + api: &WorkspaceApi, + operation: RuntimeRemovalOperation, ) -> ApiResult { if operation.state == RuntimeRemovalOperationState::Succeeded { return Ok(operation); @@ -27871,9 +29972,16 @@ async fn remove_remote_runtime( return Ok(Json(runtime_removal_response(operation))); } + let config_api = api.clone(); let config_state = api - .config_store - .load_workspace_config(&api.config.workspace_id)? + .store + .db_executor() + .run(move || { + config_api + .config_store + .load_workspace_config(&config_api.config.workspace_id) + }) + .await? .ok_or_else(|| { Error::RegistryInconsistency(format!( "Workspace {} has no active configuration", @@ -27927,22 +30035,15 @@ async fn test_runtime_connection( let checked_at = Utc::now().to_rfc3339(); let runtime = api.runtime.clone(); let ping_runtime_id = runtime_id.clone(); - let ping = tokio::task::spawn_blocking(move || runtime.ping(&ping_runtime_id)) - .await - .map_err(|_| Error::RuntimeOperationFailed { - runtime_id: runtime_id.clone(), - code: "runtime_connection_test_unavailable".to_string(), - message: "Runtime connection test could not be completed".to_string(), - })?; + let ping = api + .host_executors + .host()? + .run(move || Ok::<_, ApiError>(runtime.ping(&ping_runtime_id))) + .await?; - // The ping runs outside the store lock. A replaced/revoked connection must - // not be reported as an observation of the earlier binding. - if !api.store.workspace_runtime_binding_matches(&binding)? { - return Err(Error::RuntimeBindingConflict( - "Runtime connection changed while the connection test was running".to_string(), - ) - .into()); - } + // The ping runs outside the store lock. Recheck the entire live binding on + // the independent DB owner, never acquire its mutex on the Tokio waiter. + let binding = require_current_runtime_test_binding(&api, binding).await?; let mut result = runtime_connection_test_response(api.workspace_id(), &runtime_id, checked_at, ping); // This result describes this attempt, not permission for a future request. @@ -27954,7 +30055,12 @@ async fn test_runtime_connection( async fn get_worker_launch_options( State(api): State, ) -> ApiResult> { - Ok(Json(worker_launch_options_response(&api)?)) + // Runtime summaries can call back into DB signing/authority. Profile config + // reads are synchronous too, but this mixed helper must not occupy sole DB. + api.host_executors + .host()? + .run(move || Ok(Json(worker_launch_options_response(&api)?))) + .await } fn working_directory_request_from_repository( @@ -28133,6 +30239,27 @@ async fn create_workspace_worker_authenticated( request: CreateWorkspaceWorkerRequest, resolved_control_operation: Option, context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + create_workspace_worker_authenticated_on_host( + api, + headers, + request, + resolved_control_operation, + context.as_ref(), + ) + .await + }) + .await +} + +async fn create_workspace_worker_authenticated_on_host( + api: WorkspaceApi, + headers: HeaderMap, + request: CreateWorkspaceWorkerRequest, + resolved_control_operation: Option, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let claim_lock = request .ticket_assignment @@ -28807,30 +30934,44 @@ fn backend_resource_error_status(error: &BackendResourceError) -> StatusCode { async fn get_companion_status( State(api): State, ) -> ApiResult> { - Ok(Json(api.companion.status())) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.status()))) + .await } async fn get_companion_transcript( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(api.config.max_records).min(200); - let start = query.start.unwrap_or(0); - Ok(Json(api.companion.transcript(start, limit))) + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(api.config.max_records).min(200); + let start = query.start.unwrap_or(0); + Ok(Json(api.companion.transcript(start, limit))) + }) + .await } async fn post_companion_message( State(api): State, Json(request): Json, ) -> ApiResult> { - Ok(Json(api.companion.send_message(request))) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.send_message(request)))) + .await } async fn post_companion_cancel( State(api): State, Json(request): Json, ) -> ApiResult> { - Ok(Json(api.companion.cancel(request))) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.cancel(request)))) + .await } fn runtime_contract_restore( @@ -28929,25 +31070,30 @@ async fn get_runtime_worker( State(api): State, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let worker = api - .runtime - .worker(&worker_ref) - .map_err(|err| err.into_error())?; - let record = sync_worker_observation(&api, &worker)?; - let links = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; - let workdirs = api - .store - .list_workdir_registry(&api.config.workspace_id, 500)?; - let updated_at = record.updated_at.clone(); - let worker = - project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; - Ok(Json(server_api::RuntimeWorkerShowResponse { - worker, - updated_at, - })) + api.host_executors + .host()? + .run(move || { + let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let worker = api + .runtime + .worker(&worker_ref) + .map_err(|err| err.into_error())?; + let record = sync_worker_observation(&api, &worker)?; + let links = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; + let workdirs = api + .store + .list_workdir_registry(&api.config.workspace_id, 500)?; + let updated_at = record.updated_at.clone(); + let worker = + project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; + Ok(Json(server_api::RuntimeWorkerShowResponse { + worker, + updated_at, + })) + }) + .await } async fn restore_runtime_worker_with_context( @@ -28957,46 +31103,82 @@ async fn restore_runtime_worker_with_context( context: WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; - let mut result = worker.restore(&context, request).await?; - let projected_worker = if let Some(worker) = result.worker.as_ref() { - // A replayed terminal receipt is historical evidence, not a new Runtime observation. - let record = api - .store - .get_worker_registry(api.workspace_id(), &worker.worker)? - .ok_or_else(|| Error::UnknownWorker { - worker: worker.worker.clone(), - })?; - let links = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; - let workdirs = api - .store - .list_workdir_registry(&api.config.workspace_id, 500)?; - match project_worker_registry_projection(&api, Some(worker), &record, links, &workdirs) { - Ok(worker) => Some(worker), - Err(error) => { - result.diagnostics.push(RuntimeDiagnostic::new( - "worker_restore_current_projection_unavailable", - "warning", - sanitize_backend_error(&error.error.to_string()), - )); + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; + let result = worker.restore(&context, request).await?; + project_worker_restore_response_owned(api, runtime_id, worker_id, result).await +} + +// Post-restore stored projection is its own whole DB stage. Historical terminal +// receipts are never turned into a fresh Runtime observation or registry write. +async fn project_worker_restore_response_owned( + api: WorkspaceApi, + runtime_id: String, + worker_id: String, + mut result: crate::hosts::InternalWorkerRestoreResult, +) -> ApiResult> { + // A terminal result without a Worker has no stored projection to read. + // Do not turn that pure response into a new availability failure on DB overload. + if result.worker.is_none() { + return Ok(Json(server_api::WorkerRestoreResponse { + workspace_id: api.workspace_id().to_string(), + runtime_id, + worker_id, + result: server_api::WorkerRestoreResult { + state: result.state, + worker: None, + diagnostics: result.diagnostics.into_iter().map(Into::into).collect(), + }, + })); + } + api.store + .db_executor() + .run(move || { + let projected_worker = if let Some(worker) = result.worker.as_ref() { + // A replayed terminal receipt is historical evidence, not a new Runtime observation. + let record = api + .store + .get_worker_registry(api.workspace_id(), &worker.worker)? + .ok_or_else(|| Error::UnknownWorker { + worker: worker.worker.clone(), + })?; + let links = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; + let workdirs = api + .store + .list_workdir_registry(&api.config.workspace_id, 500)?; + match project_worker_registry_projection( + &api, + Some(worker), + &record, + links, + &workdirs, + ) { + Ok(worker) => Some(worker), + Err(error) => { + result.diagnostics.push(RuntimeDiagnostic::new( + "worker_restore_current_projection_unavailable", + "warning", + sanitize_backend_error(&error.error.to_string()), + )); + None + } + } + } else { None - } - } - } else { - None - }; - Ok(Json(server_api::WorkerRestoreResponse { - workspace_id: api.workspace_id().to_string(), - runtime_id: runtime_id.clone(), - worker_id: worker_id.clone(), - result: server_api::WorkerRestoreResult { - state: result.state, - worker: projected_worker, - diagnostics: result.diagnostics.into_iter().map(Into::into).collect(), - }, - })) + }; + Ok(Json(server_api::WorkerRestoreResponse { + workspace_id: api.workspace_id().to_string(), + runtime_id: runtime_id.clone(), + worker_id: worker_id.clone(), + result: server_api::WorkerRestoreResult { + state: result.state, + worker: projected_worker, + diagnostics: result.diagnostics.into_iter().map(Into::into).collect(), + }, + })) + }) + .await } fn reject_workdir_for_embedded_runtime(runtime_id: &str, has_workdir: bool) -> ApiResult<()> { @@ -29022,33 +31204,38 @@ async fn list_runtime_workers( AxumPath(runtime_id): AxumPath, Query(query): Query, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let (runtime_workers, source) = match query.status { - Some(RuntimeWorkersStatusFilter::Stopped) => ( - api.runtime - .list_stopped_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry_stopped", - ), - None => ( - api.runtime - .list_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry", - ), - }; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::RuntimeWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: source.to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let (runtime_workers, source) = match query.status { + Some(RuntimeWorkersStatusFilter::Stopped) => ( + api.runtime + .list_stopped_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry_stopped", + ), + None => ( + api.runtime + .list_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry", + ), + }; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::RuntimeWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: source.to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) + }) + .await } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -29449,6 +31636,25 @@ async fn create_runtime_worker_authenticated( AxumPath(runtime_id): AxumPath, Json(request): Json, context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(api.host_executors.host()?, async move { + create_runtime_worker_authenticated_on_host( + State(api), + AxumPath(runtime_id), + Json(request), + context.as_ref(), + ) + .await + }) + .await +} + +async fn create_runtime_worker_authenticated_on_host( + State(api): State, + AxumPath(runtime_id): AxumPath, + Json(request): Json, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let mut request = worker_spawn_request_from_api(request)?; let headers = context @@ -29650,17 +31856,22 @@ async fn sync_runtime_config_bundle( AxumPath(runtime_id): AxumPath, Json(request): Json, ) -> ApiResult> { - let bundle = serde_json::from_value::(request.bundle) - .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; - let result = api - .runtime - .sync_config_bundle(&runtime_id, bundle) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) + api.host_executors + .host()? + .run(move || { + let bundle = serde_json::from_value::(request.bundle) + .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; + let result = api + .runtime + .sync_config_bundle(&runtime_id, bundle) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) + }) + .await } async fn check_runtime_config_bundle( @@ -29668,21 +31879,26 @@ async fn check_runtime_config_bundle( AxumPath((runtime_id, bundle_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - let result = api - .runtime - .check_config_bundle( - &runtime_id, - ConfigBundleRef { - id: bundle_id, - digest: query.digest, - }, - ) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) + api.host_executors + .host()? + .run(move || { + let result = api + .runtime + .check_config_bundle( + &runtime_id, + ConfigBundleRef { + id: bundle_id, + digest: query.digest, + }, + ) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) + }) + .await } #[cfg(test)] @@ -29702,7 +31918,7 @@ async fn dispatch_runtime_worker_input( worker_id: String, request: WorkerInputRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; let result = worker .input(&WorkerOperationContext::Backend, request) .await?; @@ -29713,11 +31929,16 @@ async fn delete_runtime_worker_uploaded_file( State(api): State, AxumPath((runtime_id, worker_id, artifact_id)): AxumPath<(String, String, String)>, ) -> ApiResult> { - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &artifact_id) - .map_err(|err| err.into_error())?; - Ok(Json(WorkerFileDeleteResponse { deleted: true })) + api.host_executors + .host()? + .run(move || { + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &artifact_id) + .map_err(|err| err.into_error())?; + Ok(Json(WorkerFileDeleteResponse { deleted: true })) + }) + .await } async fn runtime_worker_completions( @@ -29725,13 +31946,18 @@ async fn runtime_worker_completions( AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, Json(request): Json, ) -> ApiResult> { - let request = worker_completions_request_from_api(request)?; - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let result = api - .runtime - .worker_completions(&worker, request) - .map_err(|err| err.into_error())?; - Ok(Json(worker_completions_result_to_api(result)?)) + api.host_executors + .host()? + .run(move || { + let request = worker_completions_request_from_api(request)?; + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let result = api + .runtime + .worker_completions(&worker, request) + .map_err(|err| err.into_error())?; + Ok(Json(worker_completions_result_to_api(result)?)) + }) + .await } fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource { @@ -29792,7 +32018,7 @@ async fn worker_protocol_ws( headers: HeaderMap, ws: WebSocketUpgrade, ) -> Response { - let worker = match WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) { + let worker = match WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await { Ok(worker) => worker, Err(error) => return ApiError::from(error).into_response(), }; @@ -29822,12 +32048,39 @@ pub(crate) async fn connect_workspace_worker_protocol( worker: &RuntimeWorkerRef, input_source: Option<&protocol::AuthenticatedInputSource>, ) -> Result { + let source = workspace_worker_protocol_source(api, worker).await?; + match source { + RuntimeObservationSource::RemoteWs(config) => { + connect_remote_worker_protocol(config, input_source).await + } + RuntimeObservationSource::Embedded(source) => { + let scope = + worker_runtime::RuntimeWorkspaceScope::new(api.workspace_id(), "embedded-backend"); + connect_embedded_worker_protocol(source, scope, api.host_executors.host()?).await + } + } +} + +// Cold source discovery includes live authority/signing reads. Keep the whole +// synchronous callback on its admitted DB owner, not the Tokio WS task. +async fn workspace_worker_protocol_source( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Result { let source = match api.observation_proxy.source(worker) { Ok(source) => source, - Err(ObservationProxyError::WorkerNotFound(_)) => api - .runtime - .observation_source(worker) - .map_err(|error| error.into_error())?, + Err(ObservationProxyError::WorkerNotFound(_)) => { + let runtime = api.runtime.clone(); + let worker = worker.clone(); + api.store + .db_executor() + .run(move || { + runtime + .observation_source(&worker) + .map_err(|error| error.into_error()) + }) + .await? + } Err(error) => { return Err(Error::RuntimeOperationFailed { runtime_id: worker.runtime_id.clone(), @@ -29836,16 +32089,7 @@ pub(crate) async fn connect_workspace_worker_protocol( }); } }; - match source { - RuntimeObservationSource::RemoteWs(config) => { - connect_remote_worker_protocol(config, input_source).await - } - RuntimeObservationSource::Embedded(source) => { - let scope = - worker_runtime::RuntimeWorkspaceScope::new(api.workspace_id(), "embedded-backend"); - connect_embedded_worker_protocol(source, scope).await - } - } + Ok(source) } fn insert_authenticated_input_source_header( @@ -29908,18 +32152,25 @@ async fn connect_remote_worker_protocol( async fn connect_embedded_worker_protocol( source: crate::observation::EmbeddedRuntimeObservationSource, scope: worker_runtime::RuntimeWorkspaceScope, + executor: crate::db_executor::DbExecutor, ) -> Result { - // Capture one real execution endpoint, including its greeting/snapshot. - // The WorkerRef-wide Workspace observation bus is a separate catalogue - // channel and must not reconnect this operational session after Restore. - let mut transport = source - .runtime - .attach_worker_protocol_scoped(&scope, &source.worker_ref) - .map_err(|error| Error::RuntimeOperationFailed { - runtime_id: source.worker.runtime_id.clone(), - code: "embedded_worker_protocol_connect_failed".to_string(), - message: error.to_string(), - })?; + // Endpoint capture takes Runtime operation/state locks. Keep it, and each + // intact method (including fs-store commit/publication), off Tokio and the + // sole DB lane. A discarded reply retains the endpoint until cleanup. + let capture_source = source.clone(); + let capture_scope = scope.clone(); + let mut transport = executor + .run(move || { + capture_source + .runtime + .attach_worker_protocol_scoped(&capture_scope, &capture_source.worker_ref) + .map_err(|error| Error::RuntimeOperationFailed { + runtime_id: capture_source.worker.runtime_id.clone(), + code: "embedded_worker_protocol_connect_failed".to_string(), + message: error.to_string(), + }) + }) + .await?; Ok(spawn_worker_protocol_pump( move |mut methods, events| async move { if let Err(reason) = enqueue_worker_protocol_event(&events, transport.snapshot.clone()) @@ -29931,9 +32182,26 @@ async fn connect_embedded_worker_protocol( method = methods.recv() => { let Some(method) = method else { return WorkerProtocolEnd::Disconnected; }; let shutdown = matches!(&method, protocol::Method::Shutdown { .. }); - match source.runtime.send_connected_protocol_method_scoped( - &scope, &source.worker_ref, &transport, method, - ) { + let method_source = source.clone(); + let method_scope = scope.clone(); + let owned = executor.run(move || { + let result = method_source.runtime.send_connected_protocol_method_scoped( + &method_scope, &method_source.worker_ref, &transport, method, + ); + // Endpoint validation is synchronous too. + let terminal = result.as_ref().err().is_some_and(|error| + embedded_protocol_dispatch_is_terminal(error, &transport)); + Ok::<_, Error>((transport, result, terminal)) + }).await; + let (returned, result, terminal) = match owned { + Ok(value) => value, + Err(error) => { + let _ = enqueue_worker_protocol_event(&events, protocol_error_event(error.to_string())); + return WorkerProtocolEnd::Disconnected; + } + }; + transport = returned; + match result { Ok(direct_events) => { for event in direct_events { let terminal = matches!(event, protocol::Event::Shutdown); @@ -29942,7 +32210,6 @@ async fn connect_embedded_worker_protocol( } } Err(error) => { - let terminal = embedded_protocol_dispatch_is_terminal(&error, &transport); if let Err(reason) = enqueue_worker_protocol_event(&events, protocol_error_event(error.to_string())) { return reason; } if terminal { return WorkerProtocolEnd::Disconnected; } } @@ -29955,7 +32222,6 @@ async fn connect_embedded_worker_protocol( if let Err(reason) = enqueue_worker_protocol_event(&events, event) { return reason; } if terminal { return WorkerProtocolEnd::Disconnected; } } - // Stop, disconnect and upstream lag never cause an implicit reattach. None => return WorkerProtocolEnd::Disconnected, } } @@ -30300,23 +32566,28 @@ async fn list_host_workers( State(api): State, AxumPath(host_id): AxumPath, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let runtime_workers = api - .runtime - .list_workers_for_host(&host_id, limit) - .map_err(|err| err.into_error())?; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::HostWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) + api.host_executors + .host()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtime_workers = api + .runtime + .list_workers_for_host(&host_id, limit) + .map_err(|err| err.into_error())?; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::HostWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) + }) + .await } fn project_workspace_worker_with_attachments( @@ -30415,6 +32686,20 @@ fn workers_response(api: WorkspaceApi) -> ApiResult ApiResult { + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + run_host_workflow(api.host_executors.host()?, async move { + workspace_runtime_resources_response_on_host(&api, &workspace_id).await + }) + .await +} + +// Already-owned Runtime workflows call this directly; re-admission to Host would +// wait behind the same parents. DB binding helpers still use their own DB owner. +async fn workspace_runtime_resources_response_on_host( + api: &WorkspaceApi, + workspace_id: &str, ) -> ApiResult { let limit = api.config.max_records.min(200); let runtimes = api.runtime.list_runtimes(limit); @@ -30516,12 +32801,26 @@ async fn workspace_runtime_detail( api: &WorkspaceApi, workspace_id: &str, runtime_id: &str, +) -> ApiResult { + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + run_host_workflow(api.host_executors.host()?, async move { + workspace_runtime_detail_on_host(&api, &workspace_id, &runtime_id).await + }) + .await +} + +async fn workspace_runtime_detail_on_host( + api: &WorkspaceApi, + workspace_id: &str, + runtime_id: &str, ) -> ApiResult { let binding = api .store .get_workspace_runtime_binding(workspace_id, runtime_id) .await?; - let mut resource = workspace_runtime_resources_response(api, workspace_id) + let mut resource = workspace_runtime_resources_response_on_host(api, workspace_id) .await? .items .into_iter() @@ -32875,9 +35174,33 @@ impl From for ApiError { } } +impl From for server_api::RepositoryApiError { + fn from(error: Error) -> Self { + ApiError::from(error).into_repository_api_error() + } +} + +impl From for server_api::RuntimeManagementApiError { + fn from(error: Error) -> Self { + Self::from(server_api::RepositoryApiError::from(error)) + } +} + impl From for ApiError { fn from(error: Error) -> Self { let diagnostics = match &error { + Error::DbCapacityExceeded + | Error::DbExecutorUnavailable + | Error::DbOperationOutcomeUnknown => vec![RuntimeDiagnostic { + code: match &error { + Error::DbCapacityExceeded => "db_capacity_exceeded", + Error::DbExecutorUnavailable => "db_executor_unavailable", + _ => "db_operation_outcome_unknown", + } + .into(), + severity: HostDiagnosticSeverity::Error, + message: error.to_string(), + }], Error::WorkdirSessionOpenFailed { transport, .. } => vec![RuntimeDiagnostic { code: "workdir_session_open_failed".to_string(), severity: HostDiagnosticSeverity::Error, @@ -33101,6 +35424,8 @@ fn api_error_status(error: &Error) -> StatusCode { { StatusCode::SERVICE_UNAVAILABLE } + Error::DbCapacityExceeded | Error::DbExecutorUnavailable => StatusCode::SERVICE_UNAVAILABLE, + Error::DbOperationOutcomeUnknown => StatusCode::SERVICE_UNAVAILABLE, Error::WorkspaceSigningIdentity { .. } => StatusCode::SERVICE_UNAVAILABLE, Error::WorkdirSessionOpenFailed { .. } | Error::RuntimeOperationFailed { .. } => { StatusCode::BAD_GATEWAY @@ -33158,6 +35483,10 @@ impl IntoResponse for ApiError { } } +#[cfg(test)] +#[path = "server_db_execution_tests.rs"] +mod db_execution_tests; + #[cfg(test)] mod tests { mod assignment_delivery_tests { @@ -33184,6 +35513,7 @@ mod tests { mod drive_api_tests; mod drive_tests; mod drive_web_roundtrip_tests; + mod embedded_owned_execution_tests; mod external_checkout_tests; mod protocol_backpressure_tests; mod subject_spawn_tests; @@ -34101,6 +36431,73 @@ mod tests { ); } + #[tokio::test] + async fn cancelled_workdir_db_waiter_retains_session_lock_until_mutation_finishes() { + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let api = test_api(dir.path()).await; + let worker = RuntimeWorkerRef::new("guard-runtime", "guard-worker"); + let lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(lock.clone().lock_owned().await); + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let record = WorkdirRegistryRecord { + workspace_id: TEST_WORKSPACE_ID.to_string(), + workdir_id: "cancelled-waiter-workdir".to_string(), + display_name: Some("Accepted mutation".to_string()), + source: WorkdirRegistrySource::Repository { + runtime_id: worker.runtime_id.clone(), + repository_id: api.config.repositories[0].id.clone(), + }, + creation_selector: None, + creation_ref: None, + creation_tree: None, + current_selector: None, + current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, + materialization_status: "present".to_string(), + cleanliness: "unknown".to_string(), + created_at: now.clone(), + updated_at: now, + }; + let (started, accepted) = tokio::sync::oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let stage_api = api.clone(); + let waiter = tokio::spawn(async move { + run_worker_workdir_db_stage_locked(&stage_api, &guard, move |api| { + let _ = started.send(()); + released + .recv_timeout(std::time::Duration::from_secs(5)) + .map_err(|error| Error::Store(error.to_string()))?; + api.store.upsert_workdir_registry(&record)?; + Ok::<_, Error>(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), accepted) + .await + .unwrap() + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + // This is the same Worker lock, not an executor-capacity proxy. + let still_locked = lock.try_lock().is_err(); + release.send(()).unwrap(); + assert!( + still_locked, + "accepted mutation lost its Worker session guard" + ); + let _guard = tokio::time::timeout(std::time::Duration::from_secs(5), lock.lock()) + .await + .expect("accepted mutation did not release its Worker lock"); + let persisted = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, "cancelled-waiter-workdir") + .unwrap() + .expect("accepted mutation was cancelled with its waiter"); + assert_eq!(persisted.display_name.as_deref(), Some("Accepted mutation")); + } + #[tokio::test] async fn backend_expiry_fences_offline_grant_and_persists_cleanup_completion() { let dir = tempfile::tempdir().unwrap(); @@ -34294,10 +36691,10 @@ mod tests { inject_external_workdir_cleanup_failure(&grant.workdir_id); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = session_lock.lock().await; + let session_guard = Arc::new(session_lock.lock_owned().await); let error = tokio::time::timeout( std::time::Duration::from_secs(1), - open_current_worker_workdir_session_locked(&api, &worker, &link), + open_current_worker_workdir_session_locked(&api, &worker, &link, &session_guard), ) .await .expect("session-open expiry cleanup must not reacquire its Worker lock") @@ -34842,10 +37239,11 @@ mod tests { ); let session_lock = current_worker_session_lock(&api, &worker_ref); - let session_guard = session_lock.lock().await; - let session = open_current_worker_workdir_session_locked(&api, &worker_ref, link) - .await - .unwrap(); + let session_guard = Arc::new(session_lock.lock_owned().await); + let session = + open_current_worker_workdir_session_locked(&api, &worker_ref, link, &session_guard) + .await + .unwrap(); drop(session_guard); assert_eq!(session.capabilities(), expected_capabilities); if index == 0 { @@ -35268,11 +37666,15 @@ mod tests { workdir::WorkdirSessionCapabilities::ALL ); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = session_lock.lock().await; - let broker_session = - open_current_worker_workdir_session_locked(&api, &worker, &external_link) - .await - .unwrap(); + let session_guard = Arc::new(session_lock.lock_owned().await); + let broker_session = open_current_worker_workdir_session_locked( + &api, + &worker, + &external_link, + &session_guard, + ) + .await + .unwrap(); assert_eq!( broker_session.capabilities(), workdir::WorkdirSessionCapabilities::ALL @@ -35469,6 +37871,7 @@ mod tests { Arc>>, reject_repository_access: Arc>, lifecycle_callback: Option>, + stop_callback: Option>, next_restore_outcome: Arc>>, restore_attempts: Arc>, } @@ -35775,6 +38178,9 @@ mod tests { worker_id: &str, _request: WorkerLifecycleRequest, ) -> WorkerLifecycleResult { + if let Some(callback) = &self.stop_callback { + callback(); + } WorkerLifecycleResult { state: InternalWorkerOperationState::Accepted, worker: RuntimeWorkerRef::new(Self::RUNTIME_ID, worker_id), @@ -36448,6 +38854,112 @@ mod tests { assert_eq!(error.error, "Payload Too Large"); } + #[tokio::test(flavor = "current_thread")] + async fn accepted_stop_closes_sessions_even_when_its_waiter_is_cancelled() { + let directory = tempfile::tempdir().unwrap(); + let api = test_api(directory.path()).await; + api.worker_projection.shutdown_async().await; + api.store.db_executor().drain().await.unwrap(); + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let stopped = Arc::new(tokio::sync::Notify::new()); + let notification = stopped.clone(); + let runtime = WorkdirlessFixtureRuntime { + stop_callback: Some(Arc::new(move || notification.notify_one())), + ..Default::default() + }; + api.runtime.register_or_replace(runtime.clone()); + let identity = RuntimeWorkerRef::new( + WorkdirlessFixtureRuntime::RUNTIME_ID, + Uuid::now_v7().to_string(), + ); + api.store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: TEST_WORKSPACE_ID.into(), + worker: identity.clone(), + display_name: "stop owner fixture".into(), + profile: None, + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + runtime + .workers + .lock() + .unwrap() + .push(worker_summary_from_registry( + &api.store + .get_worker_registry(TEST_WORKSPACE_ID, &identity) + .unwrap() + .unwrap(), + )); + let session: WorkdirSessionHandle = Arc::new(workdir::LocalWorkdirSession::new( + manifest::Scope::writable(directory.path()).unwrap(), + directory.path().to_path_buf(), + )); + api.workdir_sessions + .lock() + .unwrap() + .insert_attachment(identity.clone(), session); + // An actual session fence prevents cleanup until after cancellation; + // the Runtime receipt is the synchronization event, not an elapsed delay. + let session_guard = current_worker_session_lock(&api, &identity) + .lock_owned() + .await; + let worker = + WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let waiter = tokio::spawn(async move { + worker + .stop( + &WorkerOperationContext::Backend, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), stopped.notified()) + .await + .unwrap(); + assert_eq!(tokio::spawn(async { "control" }).await.unwrap(), "control"); + api.store + .db_executor() + .run(|| Ok::<_, Error>(())) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert!( + api.workdir_sessions + .lock() + .unwrap() + .attachments + .contains_key(&identity) + ); + drop(session_guard); + tokio::time::timeout( + std::time::Duration::from_secs(2), + api.host_executors.host().unwrap().drain(), + ) + .await + .unwrap() + .unwrap(); + api.store.db_executor().drain().await.unwrap(); + assert!( + !api.workdir_sessions + .lock() + .unwrap() + .attachments + .contains_key(&identity), + "accepted Runtime stop must own session cleanup after the waiter disappears" + ); + } + #[tokio::test] async fn command_session_survives_attachment_refresh_until_worker_revocation() { let directory = tempfile::tempdir().unwrap(); @@ -36687,7 +39199,7 @@ mod tests { cached_status: "connected".to_owned(), timeout: std::time::Duration::from_secs(1), }); - SqliteWorkspaceStore::open(&api.config.database_path) + SqliteWorkspaceStore::open_for_test(&api.config.database_path) .unwrap() .upsert_workspace_runtime_binding( WorkspaceRuntimeBinding { @@ -38194,9 +40706,9 @@ mod tests { async fn workspace_api_does_not_register_a_missing_workspace() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); - let error = match WorkspaceApi::new(config, store.clone()).await { + let error = match WorkspaceApi::new_for_test(config, store.clone()).await { Ok(_) => panic!("missing Workspace registration must fail closed"), Err(error) => error, }; @@ -38213,11 +40725,12 @@ mod tests { let workspace = tempfile::tempdir().unwrap(); init_clean_git_workspace(workspace.path()); let config = test_server_config(workspace.path()); - let store = Arc::new(SqliteWorkspaceStore::open(config.database_path.clone()).unwrap()); + let store = + Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); - let api = WorkspaceApi::new(config, store).await.unwrap(); + let api = WorkspaceApi::new_for_test(config, store).await.unwrap(); let workspace_id = api.config.workspace_id.clone(); let result = scoped_start_workspace_orchestrator( @@ -38272,7 +40785,7 @@ mod tests { let workspace = tempfile::tempdir().unwrap(); init_clean_git_workspace(workspace.path()); let config = test_server_config(workspace.path()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -42816,8 +45329,8 @@ mod tests { let restored_config = test_server_config(&root) .with_embedded_runtime_store_root(root.join("restored-embedded-runtime")); let restored_store = - SqliteWorkspaceStore::open(restored_config.database_path.clone()).unwrap(); - let restored = WorkspaceApi::new(restored_config, Arc::new(restored_store)) + SqliteWorkspaceStore::open_for_test(restored_config.database_path.clone()).unwrap(); + let restored = WorkspaceApi::new_for_test(restored_config, Arc::new(restored_store)) .await .unwrap(); assert!( @@ -43265,6 +45778,7 @@ mod tests { // The receiver belongs to one mock execution, not to a Runtime WorkerRecord. // Bound protocol senders become terminal when Stop drops this execution. struct DeterministicWorkerExecution { + dispatch_hook: std::sync::Mutex>>, context: worker_runtime::execution::WorkerExecutionContext, state: std::sync::Mutex, method_tx: tokio::sync::mpsc::Sender, @@ -43284,6 +45798,7 @@ mod tests { let (method_tx, method_rx) = tokio::sync::mpsc::channel(32); let (events, _) = tokio::sync::broadcast::channel(32); Self { + dispatch_hook: std::sync::Mutex::new(None), context, state: std::sync::Mutex::new(protocol::WorkerStateSnapshot::initial()), method_tx, @@ -43307,6 +45822,9 @@ mod tests { Vec, worker_runtime::execution::WorkerExecutionResult, > { + if let Some(hook) = self.dispatch_hook.lock().unwrap().take() { + hook(); + } // Drive the actual mock receiver synchronously; no method is routed // through a WorkerRef lookup after a protocol connection is attached. let mut receiver = self.method_rx.lock().unwrap(); @@ -44751,6 +47269,17 @@ mod tests { .iter() .all(|(_, content)| { !content.contains("[Ticket item checker advisory]") }) ); + // Terminal SQL state is not actual cleanup completion. Keep this + // fixture's Tokio driver alive through accepted Host/cleanup work before + // dropping it, so a pending teardown timer cannot run on a dead runtime. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + api.host_executors.host().unwrap().drain().await.unwrap(); + api.host_executors.cleanup().unwrap().drain().await.unwrap(); + api.worker_projection.shutdown_async().await; + api.store.db_executor().drain().await.unwrap(); + }) + .await + .expect("checker fixture must finish accepted cleanup before runtime shutdown"); } #[tokio::test] @@ -45143,7 +47672,8 @@ mod tests { notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), }; - let store = Arc::new(SqliteWorkspaceStore::open(config.database_path.clone()).unwrap()); + let store = + Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); @@ -45281,7 +47811,8 @@ mod tests { drop(store); drop(execution); - let restarted_store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let restarted_store = + SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let restarted_execution = Arc::new(DeterministicExecutionBackend::default()); restarted_execution.accept_restores(); let restarted = WorkspaceApi::new_with_execution_backend( @@ -45430,7 +47961,10 @@ mod tests { protocol::WorkerRunState::Running, )) })); - assert_eq!( + // Cleanup admission precedes its durable claim. The accepted job may + // still be queued, but neither queued nor executing ownership may stop + // this Run before the exact tracked finish marker below is published. + assert!(matches!( api.store .get_backend_job_attempt( TEST_WORKSPACE_ID, @@ -45440,8 +47974,8 @@ mod tests { .unwrap() .unwrap() .worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Executing) - ); + Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Executing) + )); execution.publish_worker_state( &worker, @@ -45468,7 +48002,7 @@ mod tests { async fn backend_job_delivery_recovery_replays_same_tracked_notification_id() { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend( config.clone(), @@ -46043,7 +48577,7 @@ mod tests { if let Some(parent) = config.database_path.parent() { fs::create_dir_all(parent).unwrap(); } - SqliteWorkspaceStore::open(&config.database_path).unwrap() + SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap() } #[tokio::test] @@ -46104,7 +48638,7 @@ mod tests { .unwrap(); let mut template = test_server_config(dir.path()); template.static_assets_dir = Some(static_dir); - let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); let app = build_workspace_server_router(template, store) .await .unwrap(); @@ -46187,7 +48721,7 @@ mod tests { async fn repository_free_workspace_can_be_created_selected_and_used_then_register_repository() { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let owner = seed_test_api_token(store.as_ref(), "empty-owner"); let foreign = seed_test_api_token(store.as_ref(), "empty-foreign"); let app = build_workspace_server_router(config.clone(), store.clone()) @@ -46439,7 +48973,7 @@ mod tests { .. } = &config.auth; let expected_origin = expected_origin.clone(); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); store .upsert_account(&AccountRecord { account_id: "account-auth".to_owned(), @@ -47716,7 +50250,7 @@ mod tests { .unwrap(); let mut template = test_server_config(dir.path()); template.static_assets_dir = Some(static_dir); - let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); let token = seed_test_api_token(store.as_ref(), "two-workspaces"); let catalog = WorkspaceCatalogService::new( store.clone(), @@ -47857,7 +50391,7 @@ mod tests { let repository = dir.path().join("repository"); std::fs::create_dir_all(repository.join(".git")).unwrap(); let template = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); let app = build_workspace_server_router(template, store) .await .unwrap(); @@ -47926,7 +50460,7 @@ mod tests { async fn memory_consolidation_backlog_ignores_legacy_filesystem_staging() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -47959,7 +50493,7 @@ mod tests { async fn memory_consolidation_reuses_existing_idle_worker() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -51400,7 +53934,7 @@ mod tests { workspace_root: impl Into, ) -> (WorkspaceApi, Arc) { let config = test_server_config(workspace_root); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) @@ -51424,7 +53958,7 @@ mod tests { config.repositories[0].source = source; config.repositories[0].path = None; config.repositories[0].default_selector = Some("develop".to_string()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -51453,7 +53987,7 @@ mod tests { docs.path = Some(docs_root); docs.default_selector = Some("develop".to_string()); config.repositories.push(docs); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -51888,6 +54422,7 @@ mod tests { store, Arc::new(DeterministicExecutionBackend::default()), BackendResourceBroker::default(), + HostExecutors::default(), Some(dispatcher.clone()), ) .await @@ -52133,7 +54668,7 @@ mod tests { cached_status: "connected".to_string(), timeout: std::time::Duration::from_secs(1), }); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let owner_account_id = format!("account-{TEST_WORKSPACE_ID}"); store .upsert_account(&crate::store::AccountRecord { @@ -52433,7 +54968,8 @@ mod tests { let mut revoked = trust; revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string()); - let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap(); + let authority = + SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(); authority .upsert_workspace_runtime_binding(revoked, true) .unwrap(); @@ -53363,7 +55899,7 @@ mod tests { .unwrap() .digest ); - let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); assert_eq!( changed.digest, restarted @@ -53909,7 +56445,7 @@ mod tests { .unwrap(); assert_eq!(capability_changed.items, reattached.items); assert_ne!(capability_changed.digest, reattached.digest); - let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let (restarted_page, restarted_digest) = restarted .list_worker_workdir_links_page_with_digest( TEST_WORKSPACE_ID, @@ -53928,7 +56464,7 @@ mod tests { .unwrap() .run_to_completion(128, std::time::Duration::from_millis(1), None) .unwrap(); - let restored = SqliteWorkspaceStore::from_connection(snapshot).unwrap(); + let restored = SqliteWorkspaceStore::from_connection_for_test(snapshot).unwrap(); assert_eq!( restored .list_worker_workdir_links_page_with_digest( @@ -53973,7 +56509,7 @@ mod tests { ) .unwrap(); assert_ne!(read_write_digest, capability_changed.digest); - let reader = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let reader = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let barrier = Arc::new(std::sync::Barrier::new(2)); std::thread::scope(|scope| { let worker = &fixture.worker; @@ -55137,6 +57673,8 @@ mod tests { .unwrap(); let worker = orchestrator.worker.unwrap(); let worker = RuntimeWorkerRef::new(worker.runtime_id, worker.worker_id); + let mut stop_events = api.worker_projection.subscribe_ordered(10).unwrap(); + let _ = stop_events.take_snapshot(); api.runtime .stop_worker( &worker, @@ -55146,6 +57684,25 @@ mod tests { }, ) .unwrap(); + // A synchronous Runtime stop no longer implies that its queued SQL + // projection was published. Observe stopped before taking the removal + // baseline, rather than mistaking its Upsert for the atomic removal. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let event = stop_events.recv().await.unwrap(); + if event.changes.iter().any(|change| matches!( + change, + crate::store::WorkerCatalogChange::Upsert(record) + if record.registry.worker == worker + && record.observation.as_ref().is_some_and(|observation| + observation.worker.state == protocol::subscription::SubscriptionWorkerState::Stopped) + )) { + break; + } + } + }) + .await + .expect("stopped Runtime observation must commit before removal baseline"); let worker_root = workspace .path() .join(".test-embedded-runtime-store/workers") @@ -55807,7 +58364,7 @@ mod tests { ) .unwrap(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, other_workspace, @@ -55833,7 +58390,7 @@ mod tests { let identity = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-test").unwrap(); configure_runtime_request_auth(&mut api, &identity, "runtime-test"); - let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let memory_settings = store .get_workspace_memory_settings(TEST_WORKSPACE_ID) .unwrap(); @@ -55868,7 +58425,7 @@ mod tests { let proof = issue(); let source = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, TEST_WORKSPACE_ID, @@ -55886,7 +58443,7 @@ mod tests { .unwrap(); let proof = issue(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, TEST_WORKSPACE_ID, @@ -58033,9 +60590,9 @@ mod tests { std::fs::write(static_dir.join("index.html"), "
Yoi Workspace
").unwrap(); std::fs::write(static_dir.join("assets/app.js"), "console.log('yoi');").unwrap(); - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let mut config = test_server_config(dir.path()); - let sqlite_store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let sqlite_store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); sqlite_store .upsert_workspace(&WorkspaceRecord { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -58070,7 +60627,7 @@ mod tests { "API Ticket", ticket::TicketState::Open, ); - let sqlite_store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + let sqlite_store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); sqlite_store .upsert_workspace(&WorkspaceRecord { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -59426,10 +61983,13 @@ mod tests { for delayed in delayed_protocol_test_methods() { let (runtime, worker_ref, execution) = runtime_with_worker_backend(); let source = embedded_protocol_test_source(&runtime, &worker_ref); - let mut old = - connect_embedded_worker_protocol(source.clone(), embedded_protocol_test_scope()) - .await - .unwrap(); + let mut old = connect_embedded_worker_protocol( + source.clone(), + embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), + ) + .await + .unwrap(); assert_embedded_protocol_snapshot( next_embedded_protocol_event(&mut old).await, &worker_ref, @@ -59493,10 +62053,13 @@ mod tests { &[worker_ref.clone()] ); - let mut current = - connect_embedded_worker_protocol(source, embedded_protocol_test_scope()) - .await - .unwrap(); + let mut current = connect_embedded_worker_protocol( + source, + embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), + ) + .await + .unwrap(); assert_embedded_protocol_snapshot( next_embedded_protocol_event(&mut current).await, &worker_ref, @@ -59644,10 +62207,13 @@ mod tests { for lagged in [false, true] { let (runtime, worker_ref, execution) = runtime_with_worker_backend(); let source = embedded_protocol_test_source(&runtime, &worker_ref); - let mut bridge = - connect_embedded_worker_protocol(source.clone(), embedded_protocol_test_scope()) - .await - .unwrap(); + let mut bridge = connect_embedded_worker_protocol( + source.clone(), + embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), + ) + .await + .unwrap(); assert_embedded_protocol_snapshot( next_embedded_protocol_event(&mut bridge).await, &worker_ref, @@ -59689,6 +62255,7 @@ mod tests { let mut bridge = connect_embedded_worker_protocol( embedded_protocol_test_source(&runtime, &worker_ref), embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), ) .await .unwrap(); @@ -59749,9 +62316,13 @@ mod tests { let wrong_scope = worker_runtime::RuntimeWorkspaceScope::new("other-workspace", "local-token"); assert!( - connect_embedded_worker_protocol(source.clone(), wrong_scope.clone()) - .await - .is_err() + connect_embedded_worker_protocol( + source.clone(), + wrong_scope.clone(), + HostExecutors::default().host().unwrap() + ) + .await + .is_err() ); let (mut socket, server) = spawn_embedded_protocol_test_ws(source.clone(), wrong_scope, false).await; @@ -60594,7 +63165,7 @@ mod tests { async fn objective_mutation_endpoints_round_trip_through_workspace_authority() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let owner_account_id = format!("account-{TEST_WORKSPACE_ID}"); store .upsert_account(&crate::store::AccountRecord { diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs index 0fcb072af..33ef01ed4 100644 --- a/crates/workspace-server/src/server/subjektiv_job.rs +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -65,70 +65,85 @@ pub(super) fn require_candidate_grant( /// candidate decision. Once Idle is confirmed, stop cancels any legacy surface /// background work; the new Job regenerates it and does not infer surface success. pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult { + let job_api = api.clone(); + let subject_id = subject_id.to_owned(); let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); - let Some(owner) = api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - else { - return Ok(true); + let (worker, complete) = api + .host_executors + .host()? + .run(move || { + let api = &job_api; + let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); + let Some(owner) = api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + else { + return Ok((None, true)); + }; + let mut summary = api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())?; + summary.singleton_key = Some(owner.key.clone()); + require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; + let stopped = summary.state.eq_ignore_ascii_case("stopped"); + let idle = summary.state.eq_ignore_ascii_case("idle") + && summary + .worker_state + .as_ref() + .is_some_and(|s| s.state == protocol::WorkerState::Idle); + if !stopped && !idle { + return Ok((None, false)); + } + if api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .as_ref() + .map(|o| &o.worker) + != Some(&owner.worker) + { + return Err(Error::RepositoryConflict( + "legacy consolidation owner moved during cutover".into(), + ) + .into()); + } + if !summary.state.eq_ignore_ascii_case("stopped") { + let stopped = api + .runtime + .stop_worker( + &owner.worker, + WorkerLifecycleRequest { + reason: Some( + "Retire legacy subjektiv consolidator after foreground completion" + .into(), + ), + ticket_assignment: None, + }, + ) + .map_err(|e| e.into_error())?; + if stopped.state != InternalWorkerOperationState::Accepted { + return Ok((None, false)); + } + } + if !api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())? + .state + .eq_ignore_ascii_case("stopped") + { + return Ok((None, false)); + } + Ok::<_, ApiError>((Some(owner.worker), false)) + }) + .await?; + let Some(worker) = worker else { + return Ok(complete); }; - let mut summary = api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())?; - summary.singleton_key = Some(owner.key.clone()); - require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; - let stopped = summary.state.eq_ignore_ascii_case("stopped"); - let idle = summary.state.eq_ignore_ascii_case("idle") - && summary - .worker_state - .as_ref() - .is_some_and(|s| s.state == protocol::WorkerState::Idle); - if !stopped && !idle { - return Ok(false); - } - if api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - .as_ref() - .map(|o| &o.worker) - != Some(&owner.worker) - { - return Err(Error::RepositoryConflict( - "legacy consolidation owner moved during cutover".into(), - ) - .into()); - } - if !summary.state.eq_ignore_ascii_case("stopped") { - let stopped = api - .runtime - .stop_worker( - &owner.worker, - WorkerLifecycleRequest { - reason: Some( - "Retire legacy subjektiv consolidator after foreground completion".into(), - ), - ticket_assignment: None, - }, - ) - .map_err(|e| e.into_error())?; - if stopped.state != InternalWorkerOperationState::Accepted { - return Ok(false); - } - } - if !api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())? - .state - .eq_ignore_ascii_case("stopped") - { - return Ok(false); - } let removal = WorkerRemovalService::new(api) .execute_target_removal( api.runtime.as_ref(), - &owner.worker, + &worker, "subjektiv consolidation Job cutover", None, ) @@ -141,10 +156,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) - )) .into()); } - Ok(api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - .is_none()) + let store = api.store.clone(); + let workspace_id = api.workspace_id().to_owned(); + api.store + .db_executor() + .run(move || { + store + .current_worker_singleton_owner(&workspace_id, &key) + .map(|owner| owner.is_none()) + .map_err(ApiError::from) + }) + .await } fn bounded_candidate_batch( diff --git a/crates/workspace-server/src/server/tests/api_logging_tests.rs b/crates/workspace-server/src/server/tests/api_logging_tests.rs index e4095b829..2bac1950b 100644 --- a/crates/workspace-server/src/server/tests/api_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/api_logging_tests.rs @@ -806,7 +806,19 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc .unwrap(); }; let ((status, body, events), ()) = tokio::join!( - request_with_logs(build_router(fixture.api.clone()), request), + async { + let result = request_with_logs(build_router(fixture.api.clone()), request).await; + // A pre-dispatch rejection leaves the fake provider waiting on an + // open channel. Fail with the actual response here instead of + // hanging join! until the outer suite timeout hides that cause. + assert_eq!( + result.0, + StatusCode::BAD_GATEWAY, + "early command-start response: {}", + result.1 + ); + result + }, provider ); assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); @@ -832,3 +844,78 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc "post-dispatch failure was retried" ); } + +#[tokio::test] +async fn saturated_host_before_command_start_returns_availability_without_a_provider_operation() { + let mut fixture = manual_worker_assignment_fixture().await; + let identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) + .unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let (sender, mut commands) = tokio::sync::mpsc::channel(1); + let connection = Arc::new(ExternalProviderConnection { + grant_id: "overload-grant".into(), + workdir_id: fixture.main_workdir_id.clone(), + provider_instance_id: "overload-provider".into(), + generation: 1, + expires_at: None, + capabilities: workdir::WorkdirSessionCapabilities::ALL, + admission: Arc::new(tokio::sync::Semaphore::new(1)), + shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + sender, + }); + fixture.runtime.workdir_session_factory = Some(Arc::new(move |_| { + Ok(Arc::new(ExternalProviderWorkdirSession::new( + connection.clone(), + None, + )) as WorkdirSessionHandle) + })); + fixture + .api + .runtime + .register_or_replace(fixture.runtime.clone()); + let host = fixture.api.host_executors.host().unwrap(); + host.drain().await.unwrap(); + let mut releases = Vec::new(); // dropping senders also releases jobs on panic + for _ in 0..2 { + let (release, permit) = std::sync::mpsc::channel::<()>(); + releases.push(release); + let (started, accepted) = tokio::sync::oneshot::channel(); + host.dispatch(move || { + let _ = started.send(()); + let _ = permit.recv_timeout(std::time::Duration::from_secs(10)); + }) + .unwrap(); + accepted.await.unwrap(); + } + for _ in 0..16 { + host.dispatch(|| {}).unwrap(); + } + let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations"); + let request = runtime_source_request( + &identity, + Some(&fixture.worker.worker_id), + "POST", + &path, + serde_json::to_vec(&json!({ + "target_workdir": "checkout", + "operation": {"operation": "command_start", "request": { + "command": "not-dispatched", "timeout_secs": 1, "output_limit": 1024, + "cwd": "", "spill_dir": null, "tool_call_id": "not-dispatched" + }} + })) + .unwrap(), + ); + let (status, body, _) = request_with_logs(build_router(fixture.api.clone()), request).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert!(body.to_string().contains("db_capacity_exceeded"), "{body}"); + // The response is already final but the live fake-provider receiver remains + // Pending. An unbounded join of those two futures hid this causal rejection + // behind the original 600-second suite timeout. No process or command ran. + let provider_command = commands.recv(); + tokio::pin!(provider_command); + assert!(futures::poll!(provider_command.as_mut()).is_pending()); + drop(releases); + host.drain().await.unwrap(); + fixture.api.store.db_executor().drain().await.unwrap(); +} diff --git a/crates/workspace-server/src/server/tests/auth_logging_tests.rs b/crates/workspace-server/src/server/tests/auth_logging_tests.rs index b4b51d7bc..d5bf3334a 100644 --- a/crates/workspace-server/src/server/tests/auth_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/auth_logging_tests.rs @@ -54,7 +54,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_ configure_runtime_request_auth(&mut api, &identity, "runtime-test"); let revoked = RuntimeIdentityMaterial::generate("revoked-runtime-secret").unwrap(); configure_runtime_request_auth(&mut api, &revoked, "revoked-runtime-secret"); - SqliteWorkspaceStore::open(&api.config.database_path) + SqliteWorkspaceStore::open_for_test(&api.config.database_path) .unwrap() .revoke_workspace_runtime_binding( TEST_WORKSPACE_ID, @@ -70,7 +70,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_ let body = "body-secret"; let apps = [ build_router(api.clone()), - workspace_server_router(WorkspaceServerApi::new( + workspace_server_router(WorkspaceServerApi::new_for_test( api.config.clone(), api.store.clone(), )), @@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m &error, ) }); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert!( to_bytes(response.into_body(), usize::MAX) .await @@ -347,7 +347,7 @@ async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() { let path = format!("/api/w/{TEST_WORKSPACE_ID}/repositories"); for app in [ build_router(api.clone()), - workspace_server_router(WorkspaceServerApi::new( + workspace_server_router(WorkspaceServerApi::new_for_test( api.config.clone(), api.store.clone(), )), diff --git a/crates/workspace-server/src/server/tests/drive_tests.rs b/crates/workspace-server/src/server/tests/drive_tests.rs index 1d05ff7db..3b7ec65ca 100644 --- a/crates/workspace-server/src/server/tests/drive_tests.rs +++ b/crates/workspace-server/src/server/tests/drive_tests.rs @@ -40,7 +40,7 @@ async fn cached_drive_handle_rejects_mutation_after_deletion_reservation() { }) .await .unwrap(); - let api = WorkspaceApi::new(config.clone(), store.clone()) + let api = WorkspaceApi::new_for_test(config.clone(), store.clone()) .await .unwrap(); let cached = api.drive().await.unwrap(); @@ -104,7 +104,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature // Use a separate embedded-runtime namespace for the first API: this test // reconstructs Drive authority in-process rather than spawning/restarting // product processes (runtime observers may retain their namespace). - let first_api = WorkspaceApi::new(config.clone(), store.clone()) + let first_api = WorkspaceApi::new_for_test(config.clone(), store.clone()) .await .unwrap(); // Merely constructing a Workspace API must not create blob storage. @@ -160,7 +160,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature .unwrap() .unwrap(); drop(first_api); - let recovered = WorkspaceServerApi::new(config.clone(), store.clone()); + let recovered = WorkspaceServerApi::new_for_test(config.clone(), store.clone()); let api = recovered .api_for_workspace(&config.workspace_id) .await diff --git a/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs b/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs index 02ab40e35..71efdb137 100644 --- a/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs +++ b/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs @@ -39,10 +39,10 @@ impl RealDriveFixture { let temp = tempfile::tempdir().unwrap(); let mut config = test_server_config(temp.path()); config.repositories.clear(); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let owner = seed_test_api_token(store.as_ref(), "drive-web-owner"); let second = seed_test_api_token(store.as_ref(), "drive-web-second"); - let server_api = WorkspaceServerApi::new(config, store.clone()); + let server_api = WorkspaceServerApi::new_for_test(config, store.clone()); let server = LoopbackServer::start(workspace_server_router(server_api.clone())).await; // Exercise T-717's real repositoryless Workspace bootstrap, not a // pre-existing test Workspace with its repository merely hidden in UI. diff --git a/crates/workspace-server/src/server/tests/embedded_owned_execution_tests.rs b/crates/workspace-server/src/server/tests/embedded_owned_execution_tests.rs new file mode 100644 index 000000000..55444a031 --- /dev/null +++ b/crates/workspace-server/src/server/tests/embedded_owned_execution_tests.rs @@ -0,0 +1,197 @@ +use super::*; +use crate::db_executor::{ + DbExecutor, DbExecutorLimits, DbOperationLifetime, with_operation_lifetime, +}; +use std::{ + num::NonZeroUsize, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, + }, + time::Duration, +}; + +fn owner() -> DbExecutor { + DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap() +} +async fn observed(future: impl std::future::Future) -> T { + tokio::time::timeout(Duration::from_secs(2), future) + .await + .expect("owned protocol stage did not progress") +} +struct Release(Option>); +impl Release { + fn release(&mut self) { + if let Some(tx) = self.0.take() { + let _ = tx.send(()); + } + } +} +impl Drop for Release { + fn drop(&mut self) { + self.release(); + } +} +struct Lifetime(Arc); +struct Retained(Arc); +impl Drop for Retained { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} +impl DbOperationLifetime for Lifetime { + fn retain(&self) -> Box { + self.0.fetch_add(1, Ordering::SeqCst); + Box::new(Retained(self.0.clone())) + } +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_embedded_endpoint_capture_keeps_actual_host_budget_off_tokio() { + let (runtime, worker, execution) = runtime_with_worker_backend(); + let (started, held) = tokio::sync::oneshot::channel(); + let (tx, rx) = mpsc::channel(); + let mut release = Release(Some(tx)); + let thread = std::thread::spawn(move || { + let _guard = execution.contexts.lock().unwrap(); + let _ = started.send(()); + let _ = rx.recv_timeout(Duration::from_secs(5)); + }); + observed(held).await.unwrap(); + let executor = owner(); + let leases = Arc::new(AtomicUsize::new(0)); + let mut capture = Box::pin(with_operation_lifetime( + Arc::new(Lifetime(leases.clone())), + connect_embedded_worker_protocol( + embedded_protocol_test_source(&runtime, &worker), + embedded_protocol_test_scope(), + executor.clone(), + ), + )); + assert!( + futures::poll!(&mut capture).is_pending(), + "endpoint Runtime lock must not run on Tokio" + ); + observed(tokio::time::sleep(Duration::from_millis(1))).await; + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + drop(capture); + assert_eq!(leases.load(Ordering::SeqCst), 1); + executor.dispatch(|| {}).unwrap(); // only the one queued slot is left + let error = executor.run(|| Ok::<_, Error>(())).await.unwrap_err(); + assert!(matches!(error, Error::DbCapacityExceeded)); + release.release(); + thread.join().unwrap(); + observed(executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_embedded_shutdown_commits_fs_receipt_after_the_pump_disappears() { + let dir = tempfile::tempdir().unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + let options = worker_runtime::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "owned-protocol".into(), + display_name: None, + }; + let runtime = worker_runtime::Runtime::with_fs_store_and_execution_backend( + options.clone(), + execution.clone(), + ) + .unwrap(); + runtime.store_config_bundle(runtime_test_bundle()).unwrap(); + let worker = runtime + .create_worker_scoped(&embedded_protocol_test_scope(), runtime_create_request()) + .unwrap() + .worker_ref; + let executor = owner(); + let mut connection = connect_embedded_worker_protocol( + embedded_protocol_test_source(&runtime, &worker), + embedded_protocol_test_scope(), + executor.clone(), + ) + .await + .unwrap(); + assert!(matches!( + observed(connection.events.recv()).await.unwrap(), + protocol::Event::Snapshot { .. } + )); + let (entered, held) = tokio::sync::oneshot::channel(); + let (tx, rx) = mpsc::channel(); + let mut release = Release(Some(tx)); + let endpoint = execution + .contexts + .lock() + .unwrap() + .get(&worker) + .unwrap() + .clone(); + let gate_finished = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let finished = gate_finished.clone(); + *endpoint.dispatch_hook.lock().unwrap() = Some(Box::new(move || { + let _ = entered.send(()); + let _ = rx.recv_timeout(Duration::from_secs(5)); + finished.store(true, Ordering::SeqCst); + })); + connection + .methods + .send(protocol::Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(700), + }) + .await + .unwrap(); + observed(held).await.unwrap(); // actual Runtime method, after its durable stop admission + observed(tokio::time::sleep(Duration::from_millis(1))).await; + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + assert!( + !gate_finished.load(Ordering::SeqCst), + "watchdog bailout is not Tokio progress" + ); + connection + .termination + .terminate(WorkerProtocolEnd::Disconnected); + drop(connection); + executor.dispatch(|| {}).unwrap(); + assert!(matches!( + executor.run(|| Ok::<_, Error>(())).await.unwrap_err(), + Error::DbCapacityExceeded + )); + release.release(); + observed(executor.drain()).await.unwrap(); + assert_eq!( + runtime.worker_detail(&worker).unwrap().status, + worker_runtime::catalog::WorkerStatus::Stopped + ); + // Read the actual fs-store aggregate written by the accepted method. The + // deterministic execution fixture has no restorable Profile archive, so + // restarting it would test archive recovery rather than this commit owner. + let record: serde_json::Value = serde_json::from_slice( + &std::fs::read( + options + .root + .join("workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + record["status"], "stopped", + "cancelled pump lost its fs-store commit: {record}" + ); + assert_ne!( + record["execution_state"]["state"], + "reconciliation_required" + ); +} diff --git a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs index 7a4066b39..06759a67e 100644 --- a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs +++ b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs @@ -11,6 +11,7 @@ async fn embedded_event_overflow_terminates_without_losing_the_admitted_prefix() let mut connection = connect_embedded_worker_protocol( embedded_protocol_test_source(&runtime, &worker_ref), embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), ) .await .unwrap(); @@ -69,6 +70,7 @@ async fn embedded_protocol_delivers_snapshot_then_ordered_replies_and_live_event let mut connection = connect_embedded_worker_protocol( embedded_protocol_test_source(&runtime, &worker_ref), embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), ) .await .unwrap(); @@ -105,9 +107,13 @@ async fn embedded_protocol_delivers_snapshot_then_ordered_replies_and_live_event async fn embedded_overflow_requires_explicit_reconnect_and_stop_ends_the_new_connection() { let (runtime, worker_ref, execution) = runtime_with_worker_backend(); let source = embedded_protocol_test_source(&runtime, &worker_ref); - let mut old = connect_embedded_worker_protocol(source.clone(), embedded_protocol_test_scope()) - .await - .unwrap(); + let mut old = connect_embedded_worker_protocol( + source.clone(), + embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), + ) + .await + .unwrap(); assert_embedded_protocol_snapshot(next_embedded_protocol_event(&mut old).await, &worker_ref); tokio::time::timeout(PROTOCOL_TEST_TIMEOUT, async { for index in 0..=WORKER_PROTOCOL_EVENT_CAPACITY { @@ -127,9 +133,13 @@ async fn embedded_overflow_requires_explicit_reconnect_and_stop_ends_the_new_con // Overflow ends only the attached connection, not its Worker. Reconnection // is explicit and starts from a fresh execution snapshot, not old backlog. - let mut fresh = connect_embedded_worker_protocol(source, embedded_protocol_test_scope()) - .await - .unwrap(); + let mut fresh = connect_embedded_worker_protocol( + source, + embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), + ) + .await + .unwrap(); assert_embedded_protocol_snapshot(next_embedded_protocol_event(&mut fresh).await, &worker_ref); fresh .methods @@ -767,6 +777,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) { std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 }) ); assert!(execution.protocol_methods().is_empty()); + // SQLite jobs run on external threads; paused Tokio time cannot observe their + // progress and may auto-advance a reconnect timeout before SQL completes. + tokio::time::resume(); assert_same_worker_reconnect_and_stop(&worker, &context).await; } @@ -794,6 +807,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub let mut connection = connect_embedded_worker_protocol( embedded_protocol_test_source(&runtime, &worker_ref), embedded_protocol_test_scope(), + HostExecutors::default().host().unwrap(), ) .await .unwrap(); @@ -844,7 +858,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub } } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress() { let workspace = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs index b54c4bdad..f22bc4a9d 100644 --- a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs +++ b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs @@ -1,34 +1,45 @@ use super::*; -#[tokio::test] -async fn subject_session_callback_completes_while_create_and_deletion_are_waiting() { +#[tokio::test(flavor = "current_thread")] +async fn subject_callbacks_finish_before_waiting_host_parents_release_capacity() { let workspace = tempfile::tempdir().unwrap(); let mut api = test_api(workspace.path()).await; let identity = RuntimeIdentityMaterial::generate("runtime-test").unwrap(); configure_runtime_request_auth(&mut api, &identity, "runtime-test"); let subjektiv = open_subjektiv_store(&api).unwrap(); - let subject = subjektiv - .create_subject(crate::subjektiv::SubjectRole::new("companion").unwrap()) - .unwrap(); - let singleton_key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap(); let memory = api .config_store .get_workspace_memory_settings(TEST_WORKSPACE_ID) .unwrap(); - let reserved = api - .config_store - .reserve_worker_create( - TEST_WORKSPACE_ID, - "runtime-test", - "subject-start", - &"f".repeat(64), - Some(&singleton_key), - &memory, - ) - .unwrap(); - let worker_id = reserved.worker_id.to_string(); - let server = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); - // Reuse the fixture API, but exercise the real outer dispatcher and both auth layers. + let mut callbacks = Vec::new(); + for index in 0..2 { + let subject = subjektiv + .create_subject( + crate::subjektiv::SubjectRole::new(format!("companion-{index}")).unwrap(), + ) + .unwrap(); + let singleton_key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap(); + let reserved = api + .config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + "runtime-test", + &format!("subject-start-{index}"), + &"f".repeat(64), + Some(&singleton_key), + &memory, + ) + .unwrap(); + callbacks.push(( + subject.id, + reserved.worker_id.to_string(), + session_store::new_session_id().to_string(), + )); + } + let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); + // Real dispatcher, source proof authentication, live singleton authority and + // attribution SQL. Only Runtime's create/restore wait is modeled below; no + // product process or complete create endpoint is spawned by this route test. server .apis .lock() @@ -39,61 +50,210 @@ async fn subject_session_callback_completes_while_create_and_deletion_are_waitin build_inner_router(api.clone()), ); let gate = server.admission_for_workspace(TEST_WORKSPACE_ID).await; - let create = gate.admit(false).unwrap(); - let mut deletion = Box::pin(gate.lock_deletion()); - assert!(futures::poll!(&mut deletion).is_pending()); + let host = api.host_executors.host().unwrap(); // production 2 workers/16 queued + let mut parents = Vec::new(); + let mut finish_callbacks = Vec::new(); + for _ in 0..2 { + let create = gate.admit(false).unwrap(); + let (entered, ready) = tokio::sync::oneshot::channel(); + let (finish_callback, callback_reply) = std::sync::mpsc::channel(); + finish_callbacks.push(finish_callback); + parents.push(tokio::spawn(run_host_workflow(host.clone(), async move { + let _create = create; + entered.send(()).unwrap(); + // The actual Host parent cannot finish until its mandatory callbacks + // return. Timeout/disconnection is failure cleanup, never success. + callback_reply + .recv_timeout(std::time::Duration::from_secs(5)) + .map_err(|error| { + Error::InvalidInput(format!("Runtime callback did not complete: {error}")) + })?; + Ok::<_, Error>(()) + }))); + tokio::time::timeout(std::time::Duration::from_secs(2), ready) + .await + .unwrap() + .unwrap(); + } let app = workspace_server_router(server); let path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions"); - let session_id = session_store::new_session_id().to_string(); - let body = serde_json::to_vec(&json!({ "session_id": session_id, "create_if_missing": true })) - .unwrap(); - let response = tokio::time::timeout( - std::time::Duration::from_secs(2), - app.clone().oneshot(runtime_source_request( - &identity, - Some(&worker_id), - "POST", - &path, - body.clone(), - )), - ) - .await - .expect("Session attribution must not wait for its own Worker creation") - .unwrap(); - let status = response.status(); - let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); - assert_eq!(status, StatusCode::OK, "{response_body:?}"); - let response: Value = serde_json::from_slice(&response_body).unwrap(); - assert_eq!(response["subject_id"], subject.id); - assert_eq!(response["session_id"], session_id); - let stored = subjektiv.session_attribution(&session_id).unwrap().unwrap(); - assert_eq!(stored.worker_id, worker_id); - assert_eq!(stored.subject_id, subject.id); - - // The callback lane does not bypass authentication. - let response = tokio::time::timeout( - std::time::Duration::from_secs(2), - app.clone().oneshot( - Request::builder() - .method("POST") - .uri(&path) - .header(CONTENT_TYPE, "application/json") - .body(Body::from(body)) + let memory_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/memory"); + for (_, worker_id, _) in &callbacks { + // Resident context is also a pure Subject callback used by Runtime + // bootstrap/restore. It must not reenter the waiting Host lane either. + let response = tokio::time::timeout( + std::time::Duration::from_secs(2), + app.clone().oneshot(runtime_source_request( + &identity, + Some(worker_id), + "POST", + &memory_path, + serde_json::to_vec(&server_api::SubjektivMemoryBackendRequest { + operation: server_api::SubjektivMemoryBackendOperation::ResidentContext( + Default::default(), + ), + }) .unwrap(), - ), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + )), + ) + .await + .expect("resident context must not wait for its Host parents") + .unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(status, StatusCode::OK, "{body:?}"); + let resident: server_api::SubjektivMemoryBackendResponse = + serde_json::from_slice(&body).unwrap(); + assert!(matches!( + resident, + server_api::SubjektivMemoryBackendResponse::ResidentContext(_) + )); + } + // Resident memory is not an allowed deletion-drain callback. Preserve that + // admission fence; only the exact Session attribution route reenters below. + let mut deletion = Box::pin(gate.lock_deletion()); assert!(futures::poll!(&mut deletion).is_pending()); - drop(create); + for (subject_id, worker_id, session_id) in &callbacks { + let body = + serde_json::to_vec(&json!({ "session_id": session_id, "create_if_missing": true })) + .unwrap(); + let response = tokio::time::timeout( + std::time::Duration::from_secs(2), + app.clone().oneshot(runtime_source_request( + &identity, + Some(worker_id), + "POST", + &path, + body, + )), + ) + .await + .expect("Session attribution must not wait for its Host parents") + .unwrap(); + let status = response.status(); + let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(status, StatusCode::OK, "{response_body:?}"); + let response: Value = serde_json::from_slice(&response_body).unwrap(); + assert_eq!(response["subject_id"], *subject_id); + assert_eq!(response["session_id"], *session_id); + let stored = subjektiv.session_attribution(session_id).unwrap().unwrap(); + assert_eq!(stored.worker_id, *worker_id); + assert_eq!(stored.subject_id, *subject_id); + } + // Neither actual parent nor its admission lease was released to make these + // callbacks succeed. Only durable attribution completion releases them. + assert!(parents.iter().all(|parent| !parent.is_finished())); + assert!(futures::poll!(&mut deletion).is_pending()); + for finish_callback in finish_callbacks { + finish_callback.send(()).unwrap(); + } + for parent in parents { + parent.await.unwrap().unwrap(); + } + host.drain().await.unwrap(); + api.store.db_executor().drain().await.unwrap(); let _deletion = tokio::time::timeout(std::time::Duration::from_secs(2), deletion) .await .unwrap(); assert!(gate.admit(true).is_none()); } +#[tokio::test] +async fn consolidation_subject_callbacks_are_denied_without_runtime_lookup() { + let workspace = tempfile::tempdir().unwrap(); + let mut api = test_api(workspace.path()).await; + let identity = RuntimeIdentityMaterial::generate("runtime-test").unwrap(); + configure_runtime_request_auth(&mut api, &identity, "runtime-test"); + let subjektiv = open_subjektiv_store(&api).unwrap(); + let subject = subjektiv + .create_subject(crate::subjektiv::SubjectRole::new("companion").unwrap()) + .unwrap(); + let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{}", subject.id); + let memory = api + .config_store + .get_workspace_memory_settings(TEST_WORKSPACE_ID) + .unwrap(); + let reservation = api + .config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + "runtime-test", + "consolidation-start", + &"f".repeat(64), + Some(&key), + &memory, + ) + .unwrap(); + // runtime-test has no Worker adapter: authority denial must not try to look + // up its profile/tags through Runtime before rejecting this disallowed role. + let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); + server + .apis + .lock() + .await + .insert(TEST_WORKSPACE_ID.into(), api.clone()); + server + .routers + .lock() + .await + .insert(TEST_WORKSPACE_ID.into(), build_inner_router(api.clone())); + let app = workspace_server_router(server); + let path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions"); + let worker_id = reservation.worker_id.to_string(); + let session_id = session_store::new_session_id().to_string(); + for (callback_path, callback_body) in [ + ( + path.clone(), + serde_json::to_vec(&json!({"session_id": session_id, "create_if_missing": true})) + .unwrap(), + ), + ( + format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/memory"), + serde_json::to_vec(&server_api::SubjektivMemoryBackendRequest { + operation: server_api::SubjektivMemoryBackendOperation::ResidentContext( + Default::default(), + ), + }) + .unwrap(), + ), + ] { + let response = app + .clone() + .oneshot(runtime_source_request( + &identity, + Some(&worker_id), + "POST", + &callback_path, + callback_body, + )) + .await + .unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(status, StatusCode::FORBIDDEN, "{callback_path}: {body:?}"); + } + assert!( + subjektiv + .session_attribution(&session_id) + .unwrap() + .is_none() + ); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + json!({"session_id": session_id, "create_if_missing": true}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + #[tokio::test] async fn workspace_deletion_waits_for_session_attribution_and_fences_both_lanes() { let gate = Arc::new(WorkspaceAdmission::default()); diff --git a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs index 64ca15dc3..733df3809 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs @@ -144,7 +144,7 @@ impl LaunchFixture { async fn new(remote: bool) -> Self { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let backend = Arc::new(LaunchExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), backend.clone()) diff --git a/crates/workspace-server/src/server/tests/worker_drive_tests.rs b/crates/workspace-server/src/server/tests/worker_drive_tests.rs index 2b24e45a5..481c309ad 100644 --- a/crates/workspace-server/src/server/tests/worker_drive_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_drive_tests.rs @@ -151,7 +151,7 @@ impl Fixture { let temp = tempfile::tempdir().unwrap(); let mut config = test_server_config(temp.path()); config.repositories.clear(); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let mut api = WorkspaceApi::new_with_execution_backend( config, store, diff --git a/crates/workspace-server/src/server/tests/worker_operations_tests.rs b/crates/workspace-server/src/server/tests/worker_operations_tests.rs index 2229bdb59..574c48b8e 100644 --- a/crates/workspace-server/src/server/tests/worker_operations_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_operations_tests.rs @@ -830,7 +830,11 @@ async fn grant_replacement_while_admission_waits_cannot_use_another_grants_lock( }; let admission = handle.set_pinned(&context, true); tokio::pin!(admission); - // Poll deterministically to the held lock before replacing grant authority. + // The first poll can now be waiting for the offloaded authority read, not + // the held grant lock. Drain this fixture's actual SQL, then poll its saved + // reply to reach admission before replacing authority. + assert!(futures::poll!(admission.as_mut()).is_pending()); + api.store.db_executor().drain().await.unwrap(); assert!(futures::poll!(admission.as_mut()).is_pending()); api.store .revoke_worker_control_grant( @@ -1329,8 +1333,9 @@ async fn restore_guard_internal_stale_conflict_retires_intent_for_later_explicit ); assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); // Retirement, unlike in-memory refresh, must survive caller journal restart. - api.config_store = - Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); + api.config_store = Arc::new( + SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(), + ); let fresh = api.runtime.worker(&identity).unwrap(); let next = api .config_store @@ -1490,7 +1495,7 @@ async fn restore_guard_internal_intent_survives_client_store_restart_without_rep ); // Reopen the durable caller journal, discarding all client-side memory of request identity. api.config_store = - Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); + Arc::new(SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap()); let pinned = api .config_store .pin_internal_worker_restore_intent( @@ -1599,7 +1604,7 @@ async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_w crate::repository_source::repository_source_fingerprint(&source); config.repositories[0].source = source; config.repositories[0].path = None; - let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); execution.accept_restores(); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs index 9bfbd3364..142ae143e 100644 --- a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs @@ -56,7 +56,7 @@ impl Fixture { api.runtime.register_or_replace(runtime); let now = Arc::new(AtomicI64::new(START)); let clock = now.clone(); - let mut server = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); + let mut server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); server.proof_clock = Arc::new(move || clock.load(Ordering::SeqCst)); server .apis @@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin let f = Fixture::new().await; let mut a = f.attach("1"); let (first, reply) = f.start_read("1", &mut a).await; - let mut second = Box::pin(f.app.clone().oneshot(f.operation("1"))); - // Poll through dispatch/auth until the handler waits on its resource lock. - assert!(futures::poll!(&mut second).is_pending()); + let request = f.operation("1"); + let proof = request + .headers() + .get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER) + .unwrap() + .to_str() + .unwrap(); + let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof) + .unwrap() + .jti; + let mut second = Box::pin(f.app.clone().oneshot(request)); + // DB auth is now genuinely asynchronous. Observe durable proof consumption, + // not a guessed number of polls, before advancing the injected proof clock. + let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap(); + tokio::time::timeout(BOUND, async { + loop { + assert!(futures::poll!(&mut second).is_pending()); + let consumed: bool = connection + .query_row( + "SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)", + [&jti], + |row| row.get(0), + ) + .unwrap(); + if consumed { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("second request never authenticated before resource wait"); assert!( a.try_recv().is_err(), "same session read escaped resource exclusion" @@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin assert_json(first.await.unwrap(), StatusCode::OK).await; assert!(futures::poll!(&mut second).is_pending()); let Some(ExternalProviderCommand::Operation { response, .. }) = - tokio::time::timeout(BOUND, a.recv()).await.unwrap() + tokio::time::timeout(BOUND, async { + tokio::select! { + command = a.recv() => command, + result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()), + } + }).await.unwrap() else { panic!("expected second read") }; @@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits( }), ); let mut deletion = Box::pin(f.app.clone().oneshot(request)); - assert!(futures::poll!(&mut deletion).is_pending()); let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await; - assert!( - gate.admit(false).is_none(), - "deletion start did not fence admission" - ); + tokio::time::timeout(BOUND, async { + loop { + assert!(futures::poll!(&mut deletion).is_pending()); + if gate.admit(false).is_none() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("deletion start did not fence admission after async authentication"); let session_id = session_store::new_session_id().to_string(); let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions"); let callback = f.request( diff --git a/crates/workspace-server/src/server/worker_operations.rs b/crates/workspace-server/src/server/worker_operations.rs index 35b60148f..cce576556 100644 --- a/crates/workspace-server/src/server/worker_operations.rs +++ b/crates/workspace-server/src/server/worker_operations.rs @@ -57,6 +57,19 @@ impl WorkerOperationContext { } Ok(()) } + pub(super) async fn from_request_async( + api: &WorkspaceApi, + context: &server_api::ServerRequestContext, + ) -> std::result::Result { + let owned_api = api.clone(); + let context = context.clone(); + api.store + .db_executor() + .run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context))) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? + } + pub(super) fn from_request( api: &WorkspaceApi, context: &server_api::ServerRequestContext, @@ -162,8 +175,20 @@ impl WorkspaceWorker { source: crate::worker_source::VerifiedWorkerMutationSource, reason: &str, ) -> std::result::Result { - self.current_record() - .map_err(|error| error.error.to_string())?; + let removal = self.removal.clone(); + let identity = self.identity.clone(); + self.removal + .store + .db_executor() + .run(move || { + removal + .store + .get_worker_registry(&removal.workspace_id, &identity)? + .ok_or_else(|| Error::UnknownWorker { worker: identity }) + .map(drop) + }) + .await + .map_err(|error| error.to_string())?; self.removal .execute_async( source, @@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor } impl WorkspaceWorker { + pub(crate) async fn resolve_async( + api: &WorkspaceApi, + runtime_id: &str, + reference: &str, + ) -> Result { + let owned_api = api.clone(); + let runtime_id = runtime_id.to_owned(); + let reference = reference.to_owned(); + api.store + .db_executor() + .run(move || Self::resolve(&owned_api, &runtime_id, &reference)) + .await + } + pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result { Self::resolve_with_services( api.clone(), @@ -239,7 +278,36 @@ impl WorkspaceWorker { context: &WorkerOperationContext, permission: &str, ) -> ApiResult>> { - self.current_record()?; + let worker = self.clone(); + let initial_context = context.clone(); + let permission = permission.to_owned(); + let initial_permission = permission.clone(); + let grant = self + .api + .store + .db_executor() + .run(move || -> ApiResult> { + worker.current_record()?; + let WorkerOperationContext::WorkerControl { controller } = initial_context else { + return Ok(None); + }; + if worker + .api + .store + .get_worker_registry(worker.api.workspace_id(), &controller)? + .is_none() + { + return Err(Error::UnknownWorker { worker: controller }.into()); + } + Ok(Some(authorize_known_worker_permission( + &worker.api, + worker.api.workspace_id(), + &controller, + &worker.identity, + &initial_permission, + )?)) + }) + .await?; match context { #[cfg(test)] WorkerOperationContext::Backend => Ok(None), @@ -248,42 +316,35 @@ impl WorkspaceWorker { Ok(None) } WorkerOperationContext::WorkerControl { controller } => { - if self - .api - .store - .get_worker_registry(self.api.workspace_id(), controller)? - .is_none() - { - return Err(Error::UnknownWorker { - worker: controller.clone(), - } - .into()); - } - let grant = authorize_known_worker_permission( - &self.api, - self.api.workspace_id(), - controller, - &self.identity, - permission, - )?; + let grant = grant.expect("Worker control context resolved a grant"); + // Await admission outside the DB lane; the recheck owns the guard + // until it finishes, even if its async waiter is cancelled. let guard = worker_control_lock(&self.api, &grant.grant_id) .lock_owned() .await; - self.current_record()?; - let current_grant = authorize_known_worker_permission( - &self.api, - self.api.workspace_id(), - controller, - &self.identity, - permission, - )?; - if current_grant.grant_id != grant.grant_id { - return Err(Error::RepositoryConflict( - "Worker control grant changed while waiting for admission".into(), - ) - .into()); - } - Ok(Some(guard)) + let worker = self.clone(); + let controller = controller.clone(); + self.api + .store + .db_executor() + .run(move || -> ApiResult<_> { + worker.current_record()?; + let current_grant = authorize_known_worker_permission( + &worker.api, + worker.api.workspace_id(), + &controller, + &worker.identity, + &permission, + )?; + if current_grant.grant_id != grant.grant_id { + return Err(Error::RepositoryConflict( + "Worker control grant changed while waiting for admission".into(), + ) + .into()); + } + Ok(Some(guard)) + }) + .await } } } @@ -293,8 +354,16 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "restore").await?; - self.restore_internal(request) + let guard = self.authorize_operation(context, "restore").await?; + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || { + let _guard = guard; + worker.restore_internal(request) + }) + .await } /// Trusted Backend callers still supply an explicit observed intent. Runtime @@ -312,27 +381,40 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "stop").await?; - let result = self - .api - .runtime - .stop_worker(&self.identity, request) - .map_err(|e| e.into_error())?; - if result.state != InternalWorkerOperationState::Accepted { - return Ok(result); - } - parse_runtime_worker_id_for_registry(&self.identity.worker_id)?; - let session_lock = current_worker_session_lock(&self.api, &self.identity); - let _session_guard = session_lock.lock().await; - close_current_worker_session_locked(&self.api, &self.identity).await?; - if let Some(record) = self - .api - .store - .get_worker_registry(self.api.workspace_id(), &self.identity)? - { - sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?; - } - Ok(result) + let guard = self.authorize_operation(context, "stop").await?; + let worker = self.clone(); + // Once admitted, Runtime stop, session close and linked Workdir + // publication have one owner. Never hand an accepted stop's guards + // back to its cancellable waiter or acquire this Host lane again. + run_host_workflow(self.api.host_executors.host()?, async move { + let _control = guard; + let result = worker + .api + .runtime + .stop_worker(&worker.identity, request) + .map_err(|error| error.into_error())?; + if result.state != InternalWorkerOperationState::Accepted { + return Ok(result); + } + parse_runtime_worker_id_for_registry(&worker.identity.worker_id)?; + let _session = current_worker_session_lock(&worker.api, &worker.identity) + .lock_owned() + .await; + close_current_worker_session_locked(&worker.api, &worker.identity).await?; + if let Some(record) = worker + .api + .store + .get_worker_registry(worker.api.workspace_id(), &worker.identity)? + { + sync_linked_workdir_after_worker_stop( + &worker.api, + &worker.identity.runtime_id, + &record, + )?; + } + Ok(result) + }) + .await } pub(super) async fn cancel( @@ -340,12 +422,20 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "cancel").await?; - Ok(self - .api - .runtime - .cancel_worker(&self.identity, request) - .map_err(|e| e.into_error())?) + let guard = self.authorize_operation(context, "cancel").await?; + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || -> ApiResult<_> { + let _guard = guard; + Ok(worker + .api + .runtime + .cancel_worker(&worker.identity, request) + .map_err(|e| e.into_error())?) + }) + .await } pub(super) async fn input( @@ -358,46 +448,61 @@ impl WorkspaceWorker { } else { "send_input" }; - let _guard = self.authorize_operation(context, permission).await?; + let guard = self.authorize_operation(context, permission).await?; let session_lock = current_worker_session_lock(&self.api, &self.identity); - let _session_guard = session_lock.lock().await; - self.ensure_interactive()?; + let session_guard = session_lock.lock_owned().await; + self.ensure_interactive().await?; if matches!(context, WorkerOperationContext::WorkerControl { .. }) && request.kind == WorkerInputKind::User { request.kind = WorkerInputKind::UserIfIdle; } - Ok(self - .api - .runtime - .send_input(&self.identity, request) - .map_err(|e| e.into_error())?) + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || -> ApiResult<_> { + let (_control, _session) = (guard, session_guard); + Ok(worker + .api + .runtime + .send_input(&worker.identity, request) + .map_err(|e| e.into_error())?) + }) + .await } - fn ensure_interactive(&self) -> ApiResult<()> { - if let Some(claim) = self - .api + async fn ensure_interactive(&self) -> ApiResult<()> { + let worker = self.clone(); + self.api .store - .get_pending_ticket_assignment_operation_for_worker( - self.api.workspace_id(), - &self.identity, - )? - { - return Err(Error::TicketAssignmentConflict(format!( - "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", - claim.operation_id, - )).into()); - } - if self - .api - .store - .worker_registry_projection(self.api.workspace_id(), &self.identity)? - .and_then(|projection| projection.job) - .is_some() - { - return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); - } - Ok(()) + .db_executor() + .run(move || -> ApiResult<()> { + if let Some(claim) = worker + .api + .store + .get_pending_ticket_assignment_operation_for_worker( + worker.api.workspace_id(), + &worker.identity, + )? + { + return Err(Error::TicketAssignmentConflict(format!( + "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", + claim.operation_id, + )).into()); + } + if worker + .api + .store + .worker_registry_projection(worker.api.workspace_id(), &worker.identity)? + .and_then(|projection| projection.job) + .is_some() + { + return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); + } + Ok(()) + }) + .await } pub(super) async fn set_pinned( @@ -405,33 +510,44 @@ impl WorkspaceWorker { context: &WorkerOperationContext, pinned: bool, ) -> ApiResult { - let _guard = self.authorize_operation(context, "pin").await?; - let retention_state = if pinned { "pinned" } else { "normal" }; - let changed = self.api.worker_projection.publish_ordered(|| { - self.api - .store - .update_worker_retention( - self.api.workspace_id(), - &self.identity, - retention_state, - now_registry_timestamp().as_str(), - ) - .map(|commit| (!commit.changes.is_empty(), commit)) - })?; - if !changed { - return Err(cleanup_api_error( - &self.identity.runtime_id, - "workspace_worker_retention_unknown_worker", - "Worker is not known to the Backend registry", - )); - } - Ok(WorkerRetentionResponse { - workspace_id: self.api.workspace_id().to_owned(), - runtime_id: self.identity.runtime_id.clone(), - worker_id: self.identity.worker_id.clone(), - pinned, - retention_state: retention_state.into(), - }) + let guard = self.authorize_operation(context, "pin").await?; + let worker = self.clone(); + self.api + .store + .db_executor() + .run(move || -> ApiResult { + // Publication and its DB mutation are one owned stage. Accepted + // work retains control admission through commit and publication. + let _guard = guard; + let retention_state = if pinned { "pinned" } else { "normal" }; + let changed = worker.api.worker_projection.publish_ordered(|| { + worker + .api + .store + .update_worker_retention( + worker.api.workspace_id(), + &worker.identity, + retention_state, + now_registry_timestamp().as_str(), + ) + .map(|commit| (!commit.changes.is_empty(), commit)) + })?; + if !changed { + return Err(cleanup_api_error( + &worker.identity.runtime_id, + "workspace_worker_retention_unknown_worker", + "Worker is not known to the Backend registry", + )); + } + Ok(WorkerRetentionResponse { + workspace_id: worker.api.workspace_id().to_owned(), + runtime_id: worker.identity.runtime_id.clone(), + worker_id: worker.identity.worker_id.clone(), + pinned, + retention_state: retention_state.into(), + }) + }) + .await } /// The caller owns the full cleanup-plan digest check. The removal service @@ -611,7 +727,7 @@ impl WorkspaceWorkerMethodSender { .await?; let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity); let _session_guard = session_lock.lock().await; - self.worker.ensure_interactive()?; + self.worker.ensure_interactive().await?; let WorkerOperationContext::Browser { source, .. } = context else { return Err(Error::WorkspacePermissionDenied( "Browser Worker protocol context required".into(), diff --git a/crates/workspace-server/src/server/workspace_admission.rs b/crates/workspace-server/src/server/workspace_admission.rs index 06c94b9a8..5f8335629 100644 --- a/crates/workspace-server/src/server/workspace_admission.rs +++ b/crates/workspace-server/src/server/workspace_admission.rs @@ -72,6 +72,28 @@ pub(super) struct RequestLease { callback: bool, } +impl crate::db_executor::DbOperationLifetime for RequestLease { + fn retain(&self) -> Box { + // This is continuation of an already admitted request, not fresh admission. + // Retain its drain responsibility before transferring work to the DB owner; + // dropping/aborting the HTTP waiter must not allow deletion past live SQL. + let mut state = self + .gate + .state + .lock() + .expect("Workspace admission poisoned"); + if self.callback { + state.callbacks += 1; + } else { + state.requests += 1; + } + Box::new(RequestLease { + gate: self.gate.clone(), + callback: self.callback, + }) + } +} + impl Drop for RequestLease { fn drop(&mut self) { let mut state = self diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index 7f8c57d56..ee20767bc 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -597,7 +597,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) { let dir = tempfile::tempdir().unwrap(); init_clean_git_workspace(dir.path()); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 09a50e532..e43bac8fe 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -769,7 +769,7 @@ async fn delivery_real_fixture( DeliveryScriptedClient, ) { let config = test_server_config(root); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); @@ -1007,7 +1007,7 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi tokio::task::yield_now().await; let reopened = WorkspaceApi::new_with_execution_backend( config.clone(), - Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()), + Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()), execution.clone(), ) .await diff --git a/crates/workspace-server/src/server_db_execution_tests.rs b/crates/workspace-server/src/server_db_execution_tests.rs new file mode 100644 index 000000000..08c179e6b --- /dev/null +++ b/crates/workspace-server/src/server_db_execution_tests.rs @@ -0,0 +1,1957 @@ +//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs). +#[path = "server_public_assignment_execution_tests.rs"] +mod public_assignment; +#[path = "server_residual_execution_tests.rs"] +mod residual; +use super::*; +use crate::SqliteWorkspaceStore; +use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime}; +use rusqlite::{Connection, ffi}; +use std::{ + ffi::c_void, + future::Future, + num::NonZeroUsize, + sync::{atomic::AtomicBool, mpsc}, + thread::JoinHandle, + time::Duration as StdDuration, +}; +use tokio::sync::oneshot; + +const WORKSPACE: &str = "db-execution"; +const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects"; +const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2); +const WATCHDOG: StdDuration = StdDuration::from_secs(5); + +#[track_caller] +fn observed(future: impl Future) -> impl Future { + let caller = std::panic::Location::caller(); + async move { + tokio::time::timeout(OBSERVE_LIMIT, future) + .await + .unwrap_or_else(|_| { + panic!("request did not reach the expected DB/lifecycle state at {caller}") + }) + } +} + +struct Fixture { + // Keep temporary authority files alive until all other fixture owners drop. + server: WorkspaceServerApi, + api: WorkspaceApi, + executor: DbExecutor, + host_executor: DbExecutor, + store: Arc, + _dir: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> Self { + Self::with_repositories(Vec::new()).await + } + + async fn with_repositories(repositories: Vec) -> Self { + let dir = tempfile::tempdir().unwrap(); + let executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + // Explicit startup config: never resolve the developer's live data dir. + let config = ServerConfig { + workspace_id: WORKSPACE.into(), + workspace_display_name: "DB execution fixture".into(), + workspace_created_at: "2026-01-01T00:00:00Z".into(), + workspace_execution_root: dir.path().into(), + database_path: dir.path().join("server.db"), + data_root: dir.path().join("data"), + embedded_runtime_store_root: dir.path().join("runtime"), + static_assets_dir: None, + auth: AuthConfig::Passkey { + rp_id: "localhost".into(), + origin: "http://localhost:5173".into(), + public_base_url: "http://localhost:5173".into(), + cookie_name: "db_execution_session".into(), + }, + max_records: 100, + repositories, + runtime_event_sources: Vec::new(), + remote_runtime_sources: Vec::new(), + backend_base_url: Some("http://localhost:8787".into()), + }; + let store = Arc::new( + SqliteWorkspaceStore::open_for_test(&config.database_path) + .unwrap() + .with_db_executor(executor.clone()), + ); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + for repository in &config.repositories { + store + .upsert_repository(&crate::store::RepositoryRecord { + workspace_id: WORKSPACE.into(), + repository_id: repository.id.clone(), + repository_key: repository.repository_key.clone(), + kind: "git".into(), + provider: Some(repository.provider.clone()), + source: repository.source.clone(), + default_ref: repository.default_selector.clone(), + source_fingerprint: repository.source_fingerprint.clone(), + observed_status: repository.observed_status.clone(), + observed_at: repository.observed_at.clone(), + created_at: config.workspace_created_at.clone(), + updated_at: config.workspace_created_at.clone(), + }) + .unwrap(); + } + store + .upsert_user(&UserRecord { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + created_at: config.workspace_created_at.clone(), + updated_at: config.workspace_created_at.clone(), + }) + .unwrap(); + store + .create_browser_session(&crate::store::BrowserSessionRecord { + session_id: "fixture-session".into(), + token_hash: crate::auth::token_hash("fixture-browser-token"), + user_id: "fixture-user".into(), + created_at: config.workspace_created_at.clone(), + expires_at: "2999-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + // Explicit small Host budget proves the actual mixed mutation owner; + // the production/default fixture limits remain 2 workers/16 queued. + let host_executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let host_executors = HostExecutors { + host: Arc::new(std::sync::OnceLock::from(Some(host_executor.clone()))), + ..Default::default() + }; + let api = + WorkspaceApi::new_with_host_executors(config.clone(), store.clone(), host_executors) + .await + .unwrap(); + let server = WorkspaceServerApi::new_for_test(config, store.clone()); + // Cache only the real inner router; avoid starting the unrelated hook. + server + .apis + .lock() + .await + .insert(WORKSPACE.into(), api.clone()); + server + .routers + .lock() + .await + .insert(WORKSPACE.into(), build_inner_router(api.clone())); + executor.drain().await.unwrap(); + Self { + server, + api, + executor, + host_executor, + store, + _dir: dir, + } + } + + fn request(method: &str, path: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header("cookie", "db_execution_session=fixture-browser-token") + .header(ORIGIN, "http://localhost:5173") + .body(axum::body::Body::from(body.to_string())) + .unwrap() + } + + fn subject_request(role: &str) -> Request { + Self::request( + "POST", + SUBJECTS, + serde_json::json!({ "role": role, "behavior_md": "" }), + ) + } + + fn dispatch(&self, request: Request) -> tokio::task::JoinHandle { + let server = self.server.clone(); + tokio::spawn(async move { dispatch_workspace_request(State(server), request).await }) + } + + // A pre-authenticated generated router isolates handler admission from the + // earlier browser-session DB read. Full authentication/dispatch is exercised + // separately by the cancellation and warm-Feature tests below. + fn inner_router(&self) -> Router { + build_inner_router(self.api.clone()).layer(Extension(RequestActor { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + auth_method: ActorAuthMethod::BrowserSession, + })) + } + + fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) { + let database = self + .api + .feature_storage + .open(&self.api.subjektiv_registration) + .unwrap(); + // Initialize the trusted scope, then obtain SQLite's actual file path + // from the managed handle rather than guessing the Feature directory. + crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap(); + let path = database + .with_connection(|conn| { + Ok(PathBuf::from(conn.query_row( + "SELECT file FROM pragma_database_list WHERE name = 'main'", + [], + |row| row.get::<_, String>(0), + )?)) + }) + .unwrap(); + (database, path) + } +} + +/// SQLite write ownership lives on an OS thread, including a bounded watchdog. +/// Drop releases and joins it even when an assertion panics on the async side. +struct WriteLock { + release: Option>, + thread: Option>, + released: Arc, +} + +impl WriteLock { + fn acquire(path: PathBuf) -> Self { + let (ready, held) = mpsc::sync_channel(1); + let (release, permit) = mpsc::channel(); + let released = Arc::new(AtomicBool::new(false)); + let flag = released.clone(); + let thread = std::thread::spawn(move || { + let conn = Connection::open(path).unwrap(); + conn.execute_batch("BEGIN IMMEDIATE").unwrap(); + ready.send(()).unwrap(); + let _ = permit.recv_timeout(WATCHDOG); + conn.execute_batch("ROLLBACK").unwrap(); + flag.store(true, Ordering::SeqCst); + }); + held.recv_timeout(WATCHDOG) + .expect("SQLite lock holder did not acquire its transaction"); + Self { + release: Some(release), + thread: Some(thread), + released, + } + } + + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} +impl Drop for WriteLock { + fn drop(&mut self) { + self.release(); + } +} + +struct BusyGate { + entered: Option>, + resume: mpsc::Receiver<()>, +} + +/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's +/// userdata API can, without process globals or changing production hooks. +enum BusyDatabase { + Feature(crate::FeatureDatabase), + Store(Arc), +} +impl BusyDatabase { + fn access(&self, access: impl FnOnce(&Connection)) { + match self { + Self::Feature(database) => database + .with_connection(|conn| { + access(conn); + Ok(()) + }) + .unwrap(), + Self::Store(store) => store + .with_conn(|conn| { + access(conn); + Ok(()) + }) + .unwrap(), + } + } +} +struct BusyObservation { + database: BusyDatabase, + gate: Box, + resume: Option>, +} +impl BusyObservation { + fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) { + Self::install_on(BusyDatabase::Feature(database)) + } + fn install_store(store: Arc) -> (Self, oneshot::Receiver<()>) { + Self::install_on(BusyDatabase::Store(store)) + } + fn install_on(database: BusyDatabase) -> (Self, oneshot::Receiver<()>) { + unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 { + // SAFETY: owned Box remains registered until Drop unregisters it, + // serialized by the managed connection mutex. + let gate = unsafe { &mut *context.cast::() }; + if let Some(entered) = gate.entered.take() { + let _ = entered.send(()); + return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok()); + } + 0 + } + let (entered, waiting) = oneshot::channel(); + let (resume, permit) = mpsc::channel(); + let mut gate = Box::new(BusyGate { + entered: Some(entered), + resume: permit, + }); + database.access(|conn| { + // SAFETY: handle belongs to this connection; the Box is stable. + let code = unsafe { + ffi::sqlite3_busy_handler( + conn.handle(), + Some(busy), + (&mut *gate as *mut BusyGate).cast(), + ) + }; + assert_eq!(code, ffi::SQLITE_OK); + }); + ( + Self { + database, + gate, + resume: Some(resume), + }, + waiting, + ) + } + fn resume(&mut self) { + if let Some(resume) = self.resume.take() { + let _ = resume.send(()); + } + } +} +impl Drop for BusyObservation { + fn drop(&mut self) { + self.resume(); + self.database.access(|conn| { + // SAFETY: unregister before the boxed userdata is freed. + unsafe { + ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut()); + } + }); + let _ = &self.gate; + } +} + +// A handler with one DB stage can expose its admission without adding a test +// hook to production. Polling its future to Pending proves run accepted it. +struct AdmissionObservation(Mutex>>); +impl DbOperationLifetime for AdmissionObservation { + fn retain(&self) -> Box { + if let Some(accepted) = self.0.lock().unwrap().take() { + let _ = accepted.send(()); + } + Box::new(()) + } +} +fn admission_observation() -> (Arc, oneshot::Receiver<()>) { + let (accepted, receiver) = oneshot::channel(); + ( + Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))), + receiver, + ) +} + +struct StateCheckSaturation { + executor: DbExecutor, + stages: std::sync::atomic::AtomicUsize, + releases: Arc>>>, +} +impl DbOperationLifetime for StateCheckSaturation { + fn retain(&self) -> Box { + if self.stages.fetch_add(1, Ordering::SeqCst) == 1 { + // Authentication has completed; saturate just before the lifecycle + // lookup is admitted. The callback's own dispatch admissions are + // distinct stages and do not recurse into this branch. + let (started, running) = mpsc::channel(); + let (release, permit) = mpsc::channel(); + self.executor + .dispatch(move || { + let _ = started.send(()); + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + running + .recv_timeout(WATCHDOG) + .expect("state-check blocker did not start"); + self.releases.lock().unwrap().push(release); + let (release, permit) = mpsc::channel(); + self.executor + .dispatch(move || { + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + self.releases.lock().unwrap().push(release); + } + Box::new(()) + } +} +struct ReleaseStateCheckBlockers(Arc>>>); +impl Drop for ReleaseStateCheckBlockers { + fn drop(&mut self) { + self.0.lock().unwrap().clear(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() { + let fixture = Fixture::new().await; + let releases = Arc::new(Mutex::new(Vec::new())); + let release = ReleaseStateCheckBlockers(releases.clone()); + let source = Arc::new(StateCheckSaturation { + executor: fixture.executor.clone(), + stages: std::sync::atomic::AtomicUsize::new(0), + releases, + }); + let router = Router::new() + .route( + SUBJECTS, + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .layer(axum::middleware::from_fn_with_state( + fixture.api.clone(), + authorize_workspace_api_request, + )); + let response = observed(crate::db_executor::with_operation_lifetime( + source, + router.oneshot(Fixture::subject_request("not-executed")), + )) + .await + .unwrap(); + drop(release); + observed(fixture.executor.drain()).await.unwrap(); + let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; + assert!( + !body.to_string().contains("Workspace is deleting"), + "{body}" + ); +} + +async fn runtime_progress_while_locked(lock: &WriteLock) { + let control = tokio::spawn(async { "control progressed" }); + assert_eq!(observed(control).await.unwrap(), "control progressed"); + // The timer is the contract, not a guessed wait for request completion. + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !lock.released.load(Ordering::SeqCst), + "SQLite watchdog released the lock before runtime progress" + ); +} +async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value { + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); + serde_json::from_slice(&bytes).expect("typed JSON response") +} +fn subject_count(path: &Path, role: &str) -> i64 { + Connection::open(path) + .unwrap() + .query_row( + "SELECT count(*) FROM subjects WHERE role = ?1", + [role], + |row| row.get(0), + ) + .unwrap() +} + +#[tokio::test(flavor = "current_thread")] +async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() { + let fixture = Fixture::new().await; + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let (lifetime, accepted) = admission_observation(); + let request = Fixture::request( + "POST", + "/api/w/db-execution/tickets", + serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(), + ); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture.inner_router().oneshot(request), + )); + assert!(futures::poll!(&mut response).is_pending()); + observed(accepted).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + futures::poll!(&mut response).is_pending(), + "mutation responded before the SQLite lock was released" + ); + let reader = Connection::open(&fixture.api.config.database_path).unwrap(); + let count: i64 = reader + .query_row( + "SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1", + [WORKSPACE], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 0, "Ticket must not be visible before commit"); + lock.release(); + let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; + assert!(body["id"].is_string(), "{body}"); + // Ticket dispatch can call Runtime callbacks, so its operation owner is + // the separate bounded host executor, not the store executor. + observed(fixture.api.host_executors.host().unwrap().drain()) + .await + .unwrap(); + fixture.executor.drain().await.unwrap(); + let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap(); + assert_eq!(count, 1, "one accepted HTTP mutation must commit once"); +} + +#[tokio::test(flavor = "current_thread")] +async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + drop(database); // FeatureStorage retains Weak handles: the route must cold-open. + let mut lock = WriteLock::acquire(path); + let (lifetime, accepted) = admission_observation(); + let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture.inner_router().oneshot(request), + )); + assert!(futures::poll!(&mut response).is_pending()); + observed(accepted).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + futures::poll!(&mut response).is_pending(), + "cold open must wait for its SQLite scope transaction" + ); + lock.release(); + let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; + assert_eq!(body["items"], serde_json::json!([]), "{body}"); + fixture.executor.drain().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let response = fixture.dispatch(Fixture::subject_request("locked-subject")); + observed(waiting).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + !response.is_finished(), + "HTTP response must not publish an uncommitted subject" + ); + assert_eq!(subject_count(&path, "locked-subject"), 0); + lock.release(); + busy.resume(); + let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await; + assert_eq!(body["role"], "locked-subject", "{body}"); + assert!( + body["memory_fingerprint"].is_string(), + "response assembly must finish in the admitted mutation operation: {body}" + ); + fixture.host_executor.drain().await.unwrap(); + assert_eq!(subject_count(&path, "locked-subject"), 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let response = fixture.dispatch(Fixture::subject_request("cancelled-subject")); + observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling. + response.abort(); + assert!(observed(response).await.unwrap_err().is_cancelled()); + let admission = fixture.server.admission_for_workspace(WORKSPACE).await; + let mut deletion = Box::pin(admission.lock_deletion()); + assert!( + futures::poll!(&mut deletion).is_pending(), + "aborting HTTP must not drop the accepted SQL RequestLease" + ); + assert!( + admission.admit(false).is_none(), + "deletion must fence fresh requests" + ); + assert_eq!(subject_count(&path, "cancelled-subject"), 0); + lock.release(); + busy.resume(); + fixture.host_executor.drain().await.unwrap(); + let deletion = observed(deletion).await; + assert_eq!( + subject_count(&path, "cancelled-subject"), + 1, + "cancelled waiter must neither cancel nor retry accepted SQL" + ); + drop(deletion); + assert!( + admission.admit(false).is_some(), + "deletion lease cleanup must reopen admission" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let running = fixture.dispatch(Fixture::subject_request("running-subject")); + observed(waiting).await.unwrap(); + let (lifetime, accepted) = admission_observation(); + let mut queued = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture + .inner_router() + .oneshot(Fixture::subject_request("queued-subject")), + )); + assert!(futures::poll!(&mut queued).is_pending()); + observed(accepted).await.unwrap(); + drop(queued); // Accepted job still owns the only waiting slot. + let rejected = observed( + fixture + .inner_router() + .oneshot(Fixture::subject_request("rejected-subject")), + ) + .await + .unwrap(); + let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await; + let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap(); + assert!( + error.message.contains("capacity"), + "typed overload error: {body}" + ); + assert_eq!(subject_count(&path, "queued-subject"), 0); + assert_eq!(subject_count(&path, "rejected-subject"), 0); + let mut drained = Box::pin(fixture.host_executor.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + lock.release(); + busy.resume(); + response_json(observed(running).await.unwrap(), StatusCode::CREATED).await; + observed(drained).await.unwrap(); + assert_eq!( + subject_count(&path, "queued-subject"), + 1, + "cancelled accepted mutation persists exactly once" + ); + assert_eq!( + subject_count(&path, "rejected-subject"), + 0, + "overload must not execute rejected SQL" + ); + let retry = fixture.dispatch(Fixture::subject_request("after-drain")); + response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await; + fixture.host_executor.drain().await.unwrap(); + assert_eq!( + subject_count(&path, "after-drain"), + 1, + "drain must release actual-job capacity" + ); +} + +#[tokio::test(start_paused = true)] +async fn saturated_cancelled_host_jobs_keep_their_budget_but_not_other_fixture_or_cleanup_admission() + { + use std::sync::{ + Condvar, + atomic::{AtomicUsize, Ordering}, + }; + // A drop guard releases workers even when an assertion panics. + struct Release(Arc<(Mutex, Condvar)>); + impl Drop for Release { + fn drop(&mut self) { + *self.0.0.lock().unwrap() = true; + self.0.1.notify_all(); + } + } + let owner = HostExecutors::default(); + let other = HostExecutors::default(); + let host = owner.host().unwrap(); + let gate = Arc::new((Mutex::new(false), Condvar::new())); + let release = Release(gate.clone()); + let completed = Arc::new(AtomicUsize::new(0)); + let mut waiters = Vec::new(); + for _ in 0..2 { + let host = host.clone(); + let gate = gate.clone(); + let completed = completed.clone(); + let (started, accepted) = tokio::sync::oneshot::channel(); + waiters.push(tokio::spawn(async move { + host.run(move || { + started.send(()).unwrap(); + let mut released = gate.0.lock().unwrap(); + while !*released { + released = gate.1.wait(released).unwrap(); + } + completed.fetch_add(1, Ordering::SeqCst); + Ok::<_, Error>(()) + }) + .await + })); + accepted.await.unwrap(); + } + for _ in 0..16 { + let completed = completed.clone(); + host.dispatch(move || { + completed.fetch_add(1, Ordering::SeqCst); + }) + .unwrap(); + } + for waiter in waiters { + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + } + assert!(matches!( + host.run(|| Ok::<_, Error>(())).await, + Err(Error::DbCapacityExceeded) + )); + // Paused Tokio time can advance without completing or releasing actual Host + // work. Neither another fixture nor cleanup needs this saturated Host lane. + tokio::time::advance(std::time::Duration::from_secs(31)).await; + assert_eq!(completed.load(Ordering::SeqCst), 0); + other + .host() + .unwrap() + .run(|| Ok::<_, Error>(())) + .await + .unwrap(); + owner + .cleanup() + .unwrap() + .run(|| Ok::<_, Error>(())) + .await + .unwrap(); + assert!(matches!( + host.run(|| Ok::<_, Error>(())).await, + Err(Error::DbCapacityExceeded) + )); + drop(release); + host.drain().await.unwrap(); + assert_eq!(completed.load(Ordering::SeqCst), 18); + host.run(|| Ok::<_, Error>(())).await.unwrap(); + host.drain().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn restore_intent_pins_once_after_write_ownership_without_a_deferred_read_upgrade() { + let fixture = Fixture::new().await; + let store = fixture.api.config_store.clone(); + let (mut busy, waiting) = BusyObservation::install_store(store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let worker = RuntimeWorkerRef::new("restore-runtime", "restore-worker"); + let pin_store = store.clone(); + let pin_worker = worker.clone(); + let mut pinned = Box::pin(fixture.executor.run(move || { + pin_store.pin_internal_worker_restore_intent( + WORKSPACE, + &pin_worker, + "restore-domain", + Some("observed-before-dispatch"), + Some("restore-request"), + ) + })); + tokio::select! { + result = &mut pinned => panic!("restore intent must wait for write ownership before reading, not fail a deferred read upgrade: {result:?}"), + arrived = observed(waiting) => arrived.unwrap(), + } + runtime_progress_while_locked(&lock).await; + assert!(futures::poll!(&mut pinned).is_pending()); + lock.release(); + busy.resume(); + let request = observed(pinned).await.unwrap(); + assert_eq!(request.request_id, "restore-request"); + assert_eq!( + request.expected_observation_token, + "observed-before-dispatch" + ); + fixture.executor.drain().await.unwrap(); + let saved = store + .get_internal_worker_restore_intent(WORKSPACE, &worker, &request.request_id) + .unwrap() + .unwrap(); + assert_eq!(saved.request_id, request.request_id); + assert_eq!( + saved.expected_observation_token, + request.expected_observation_token + ); + let count: i64 = store + .with_conn(|conn| { + Ok(conn.query_row( + "SELECT count(*) FROM worker_restore_intents WHERE workspace_id=?1 AND request_id=?2", + rusqlite::params![WORKSPACE, request.request_id], |row| row.get(0) + )?) + }) + .unwrap(); + assert_eq!( + count, 1, + "one admitted intent must persist exactly once, with no operation retry" + ); +} + +fn install_remote_fixture( + fixture: &Fixture, + runtime_id: &str, + base_url: &str, + signed: bool, +) -> crate::store::WorkspaceRuntimeBinding { + let identity = worker_runtime::auth::RuntimeIdentityMaterial::generate(runtime_id).unwrap(); + fixture + .store + .upsert_workspace_runtime_binding( + crate::store::WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: runtime_id.into(), + display_name: "Held Runtime".into(), + base_url: base_url.into(), + public_key: identity.public_key, + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: if signed { + crate::store::WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity + } else { + crate::store::WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer + }, + created_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + revoked_at: None, + }, + false, + ) + .unwrap(); + let binding = fixture + .store + .get_workspace_runtime_binding(WORKSPACE, runtime_id) + .unwrap() + .unwrap(); + // This narrow fixture exercises the provider's own live authority callback; + // it does not start the unrelated subscription/projection reconnect producer. + fixture.api.runtime.set_runtime_binding_gate(|_| true); + let mut config = crate::hosts::RemoteRuntimeConfig::new( + runtime_id, + "Held Runtime", + base_url, + Some("must-not-fall-back".into()), + ) + .with_timeout(WATCHDOG); + if signed { + fixture + .api + .signing_identities + .provision_existing(WORKSPACE, &format!("account-{WORKSPACE}")) + .unwrap(); + config.workspace_authorization = Some(WorkspaceRuntimeAuthorization::new( + fixture.store.clone(), + fixture.api.signing_identities.clone(), + "http://localhost:8787", + binding.clone(), + )); + } + fixture.api.runtime.register_or_replace( + RemoteWorkerRuntime::new(config, WORKSPACE.into(), "http://localhost:8787".into()).unwrap(), + ); + binding +} + +#[tokio::test(flavor = "current_thread")] +async fn cold_protocol_source_signing_waits_off_tokio_and_never_falls_back_on_overload_or_revocation() + { + let fixture = Fixture::new().await; + let binding = install_remote_fixture(&fixture, "protocol-runtime", "http://127.0.0.1:9", true); + let worker = RuntimeWorkerRef::new(&binding.runtime_id, "source-worker"); + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let writer = fixture.store.clone(); + fixture + .executor + .dispatch(move || { + writer + .mark_api_token_used("missing-token", "2026-01-01T00:00:00Z") + .unwrap() + }) + .unwrap(); + observed(waiting).await.unwrap(); // SQL holds the actual shared connection mutex. + let mut source = Box::pin(workspace_worker_protocol_source(&fixture.api, &worker)); + assert!(futures::poll!(&mut source).is_pending()); + runtime_progress_while_locked(&lock).await; + assert!(futures::poll!(&mut source).is_pending()); + assert!( + matches!( + workspace_worker_protocol_source(&fixture.api, &worker).await, + Err(Error::DbCapacityExceeded) + ), + "overload must not return an unsigned or legacy-bearer source" + ); + lock.release(); + busy.resume(); + let RuntimeObservationSource::RemoteWs(config) = observed(source).await.unwrap() else { + panic!("remote source expected") + }; + let first = config + .bearer_token + .expect("fresh Workspace capability is required"); + assert_ne!(first, "must-not-fall-back"); + let RuntimeObservationSource::RemoteWs(second) = + workspace_worker_protocol_source(&fixture.api, &worker) + .await + .unwrap() + else { + panic!("remote source expected") + }; + assert_ne!( + first, + second.bearer_token.unwrap(), + "cold lookup must sign a fresh request, not cache authentication success" + ); + fixture + .store + .revoke_workspace_runtime_binding(WORKSPACE, &binding.runtime_id, "2026-01-02T00:00:00Z") + .unwrap(); + let error = workspace_worker_protocol_source(&fixture.api, &worker) + .await + .err() + .expect("revoked authority cannot produce any source"); + assert!( + matches!(error, Error::RuntimeOperationFailed { ref code, .. } if code == "workspace_runtime_authorization_stale"), + "{error:?}" + ); + fixture.executor.drain().await.unwrap(); +} + +// Loopback fake at the Session/Worker Runtime HTTP boundary. It owns one reply, +// released by an explicit event; Drop unblocks accept/read and joins on failure. +struct HeldRuntimeReply { + address: std::net::SocketAddr, + entered: Option>, + release: Option>, + upstream_released: Arc, + thread: Option>, +} +impl HeldRuntimeReply { + fn new() -> Self { + Self::with_held_path(None) + } + fn for_path(path: &str) -> Self { + Self::with_held_path(Some(path.to_owned())) + } + fn with_held_path(path: Option) -> Self { + use std::io::Write; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (entered, received) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let upstream_released = Arc::new(AtomicBool::new(false)); + let flag = upstream_released.clone(); + let thread = std::thread::spawn(move || { + loop { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + stream.set_read_timeout(Some(WATCHDOG)).unwrap(); + let Ok(Some(request)) = read_runtime_fixture_request(&mut stream) else { + return; + }; + if path.as_ref().is_some_and(|path| !request.contains(path)) { + // Ready's best-effort notification lookup precedes its detail + // source observation. Deny it promptly; hold only the named stage. + let _ = stream.write_all(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + continue; + } + let _ = entered.send(request); + let _ = permit.recv_timeout(WATCHDOG); + flag.store(true, Ordering::SeqCst); + let _ = stream.write_all(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + break; + } + }); + Self { + address, + entered: Some(received), + release: Some(release), + upstream_released, + thread: Some(thread), + } + } + fn finish(&mut self) { + self.release.take(); + let _ = std::net::TcpStream::connect(self.address); + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} +impl Drop for HeldRuntimeReply { + fn drop(&mut self) { + self.finish(); + } +} + +// Read one complete request before selecting a held path or denying a prior +// notification. TCP reads are not HTTP frame boundaries; replying while a POST +// body remains unread can also reset the connection instead of delivering 503. +fn read_runtime_fixture_request(reader: impl std::io::Read) -> std::io::Result> { + use std::io::{BufRead, Read}; + let mut reader = std::io::BufReader::new(reader); + let mut line = String::new(); + if reader.read_line(&mut line)? == 0 { + return Ok(None); + } + let request = line.trim_end().to_owned(); + let mut header_bytes = line.len(); + let mut body_len = 0; + loop { + line.clear(); + if reader.read_line(&mut line)? == 0 { + return Err(std::io::ErrorKind::UnexpectedEof.into()); + } + header_bytes += line.len(); + if header_bytes > 16384 { + return Err(std::io::ErrorKind::InvalidData.into()); + } + if line == "\r\n" { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + body_len = value + .trim() + .parse::() + .map_err(|_| std::io::ErrorKind::InvalidData)?; + if body_len > 1024 * 1024 { + return Err(std::io::ErrorKind::InvalidData.into()); + } + } + if name.eq_ignore_ascii_case("transfer-encoding") { + return Err(std::io::ErrorKind::InvalidData.into()); + } + } + } + let mut body = vec![0; body_len]; + reader.read_exact(&mut body)?; + Ok(Some(request)) +} + +#[test] +fn held_runtime_fixture_consumes_fragmented_headers_and_body_before_selecting_the_path() { + struct Fragmented(std::io::Cursor>); + impl std::io::Read for Fragmented { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let length = buffer.len().min(2); + std::io::Read::read(&mut self.0, &mut buffer[..length]) + } + } + let request = b"POST /v1/repository-refs/observe HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\nnotes"; + let mut reader = Fragmented(std::io::Cursor::new(request.to_vec())); + assert_eq!( + read_runtime_fixture_request(&mut reader) + .unwrap() + .as_deref(), + Some("POST /v1/repository-refs/observe HTTP/1.1") + ); + assert_eq!(reader.0.position(), request.len() as u64); + let truncated = &request[..request.len() - 1]; + assert_eq!( + read_runtime_fixture_request(truncated).unwrap_err().kind(), + std::io::ErrorKind::UnexpectedEof + ); +} + +struct CountJobLifetime(Arc); +struct CountJobLease(Arc); +impl DbOperationLifetime for CountJobLifetime { + fn retain(&self) -> Box { + self.0.fetch_add(1, Ordering::SeqCst); + Box::new(CountJobLease(self.0.clone())) + } +} +impl Drop for CountJobLease { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn subjektiv_runtime_waits_leave_db_and_control_free_and_cancel_retains_the_actual_host_lease() + { + // Different mixed helpers carry this rule: Session Read/Search, subject + // projection, and legacy consolidation scope authorization (even refusal). + for kind in ["read", "search", "subject", "legacy_scope"] { + let fixture = Fixture::new().await; + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "subject-runtime", + &format!("http://{}", reply.address), + false, + ); + let Json(subject) = scoped_create_subjektiv_subject( + State(fixture.api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: WORKSPACE.into(), + }), + Json(server_api::SubjektivSubjectCreateRequest { + role: "network subject".into(), + behavior_md: String::new(), + }), + ) + .await + .unwrap() + .1; + let singleton = if kind == "legacy_scope" { + format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{}", subject.id) + } else { + crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap() + }; + let memory = fixture + .store + .get_workspace_memory_settings(WORKSPACE) + .unwrap(); + let reservation = fixture + .store + .reserve_worker_create( + WORKSPACE, + "subject-runtime", + "network-fixture", + "network-fixture-fingerprint", + Some(&singleton), + &memory, + ) + .unwrap(); + let worker = RuntimeWorkerRef::new("subject-runtime", reservation.worker_id.to_string()); + fixture + .store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: WORKSPACE.into(), + worker: worker.clone(), + display_name: "network fixture".into(), + profile: None, + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + }) + .unwrap(); + fixture + .store + .complete_worker_create_reservation(WORKSPACE, reservation.worker_id) + .unwrap(); + open_subjektiv_store(&fixture.api) + .unwrap() + .record_session_attribution( + crate::subjektiv::SubjectSessionAttribution::new( + subject.id.clone(), + worker.runtime_id.clone(), + worker.worker_id.clone(), + "session", + ) + .unwrap(), + ) + .unwrap(); + fixture + .store + .create_worker_control_grant(&WorkerControlGrantRecord { + workspace_id: WORKSPACE.into(), + grant_id: "session-observe".into(), + controller: worker.clone(), + subject: worker.clone(), + relation: "subject_session".into(), + origin: "test".into(), + permissions: vec!["observe".into()], + operation_id: "session-observe".into(), + created_at: "2026-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + let context = server_api::ServerRequestContext { + actor: None, + worker_source: None, + runtime_source: Some(server_api::ServerRuntimeSource { + authentication_binding_id: "fixture-binding".into(), + runtime_id: worker.runtime_id, + worker_id: Some(worker.worker_id), + }), + origin: None, + transport_headers: Vec::new(), + }; + let counts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lifetime = Arc::new(CountJobLifetime(counts.clone())); + let api = fixture.api.clone(); + let task = tokio::spawn(crate::db_executor::with_operation_lifetime( + lifetime, + async move { + if kind == "subject" { + scoped_get_subjektiv_subject( + State(api), + AxumPath(ScopedSubjektivSubjectPath { + workspace_id: WORKSPACE.into(), + subject_id: subject.id, + }), + ) + .await + .map(|_| ()) + } else { + let operation = match kind { + "read" => server_api::SubjektivSessionBackendOperation::Read( + server_api::SubjektivSessionReadRequest { + session_id: "session".into(), + segment_id: "segment".into(), + entry_ref: "E1".into(), + mode: Default::default(), + cursor: None, + }, + ), + "search" => server_api::SubjektivSessionBackendOperation::Search( + serde_json::from_value( + serde_json::json!({"session_id":"session","query":"needle"}), + ) + .unwrap(), + ), + _ => server_api::SubjektivSessionBackendOperation::List( + serde_json::from_value(serde_json::json!({})).unwrap(), + ), + }; + scoped_subjektiv_session_backend( + State(api), + AxumPath(ScopedWorkspacePath { + workspace_id: WORKSPACE.into(), + }), + context, + Json(server_api::SubjektivSessionBackendRequest { operation }), + ) + .await + .map(|_| ()) + } + }, + )); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!( + request.contains(if kind == "read" || kind == "search" { + "/v1/session-public/" + } else { + "/v1/workers/" + }), + "{kind}: {request}" + ); + let read_store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || read_store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; // timer contract, not a wait guess + assert!( + !task.is_finished(), + "{kind} completed before upstream release" + ); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!( + counts.load(Ordering::SeqCst), + 1, + "{kind}: cancellation lost actual Host authority" + ); + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + assert_eq!( + counts.load(Ordering::SeqCst), + 0, + "{kind}: completed Host job leaked its lease" + ); + fixture.executor.drain().await.unwrap(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn runtime_registration_scan_keeps_sql_and_tokio_free_and_retains_cancelled_host_budget() { + let fixture = Fixture::new().await; + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "scan-runtime", + &format!("http://{}", reply.address), + true, + ); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let api = fixture.api.clone(); + let task = tokio::spawn(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + async move { scan_runtime_registration_workers(&api, "scan-runtime").await }, + )); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!(request.contains("/v1/workers"), "{request}"); + assert!( + !reply.upstream_released.load(Ordering::SeqCst), + "registration scan blocked Tokio until watchdog bailout" + ); + let store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + let mut queued_control = Box::pin(fixture.host_executor.run(|| Ok::<_, Error>(()))); + assert!(futures::poll!(&mut queued_control).is_pending()); + let rejected = scan_runtime_registration_workers(&fixture.api, "scan-runtime") + .await + .unwrap_err(); + response_json(rejected.into_response(), StatusCode::SERVICE_UNAVAILABLE).await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(leases.load(Ordering::SeqCst), 1); + assert!(!reply.upstream_released.load(Ordering::SeqCst)); + reply.finish(); + observed(queued_control).await.unwrap(); + observed(fixture.host_executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + fixture.executor.drain().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn runtime_management_authority_waits_off_tokio_and_retains_cancelled_db_admission() { + for stage in ["identity", "binding"] { + let fixture = Fixture::new().await; + let binding = + install_remote_fixture(&fixture, "authority-runtime", "http://127.0.0.1:9", true); + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let writer = fixture.store.clone(); + fixture + .executor + .dispatch(move || { + writer + .mark_api_token_used("missing-token", "2026-01-01T00:00:00Z") + .unwrap(); + }) + .unwrap(); + observed(waiting).await.unwrap(); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let api = fixture.api.clone(); + let expected = binding.clone(); + let mut authority = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + async move { + if stage == "identity" { + require_active_runtime_registration_identity(&api).await + } else { + require_current_runtime_test_binding(&api, expected) + .await + .map(|_| ()) + } + }, + )); + assert!( + futures::poll!(&mut authority).is_pending(), + "{stage}: authority must queue without blocking Tokio on the shared mutex" + ); + runtime_progress_while_locked(&lock).await; + let overload = if stage == "identity" { + require_active_runtime_registration_identity(&fixture.api) + .await + .unwrap_err() + } else { + require_current_runtime_test_binding(&fixture.api, binding.clone()) + .await + .unwrap_err() + }; + response_json(overload.into_response(), StatusCode::SERVICE_UNAVAILABLE).await; + drop(authority); + assert_eq!( + leases.load(Ordering::SeqCst), + 1, + "{stage}: cancelled DB authority lost its lease" + ); + lock.release(); + busy.resume(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + require_active_runtime_registration_identity(&fixture.api) + .await + .unwrap(); + assert_eq!( + require_current_runtime_test_binding(&fixture.api, binding.clone()) + .await + .unwrap() + .binding_id, + binding.binding_id + ); + fixture + .store + .revoke_workspace_runtime_binding( + WORKSPACE, + &binding.runtime_id, + "2026-01-02T00:00:00Z", + ) + .unwrap(); + let stale = require_current_runtime_test_binding(&fixture.api, binding) + .await + .unwrap_err(); + response_json(stale.into_response(), StatusCode::CONFLICT).await; + } +} + +#[tokio::test(flavor = "current_thread")] +async fn runtime_connection_post_ping_sql_waits_off_tokio_while_host_control_progresses() { + let fixture = Fixture::new().await; + let mut reply = HeldRuntimeReply::new(); + let binding = install_remote_fixture( + &fixture, + "ping-runtime", + &format!("http://{}", reply.address), + true, + ); + let mut test = Box::pin(test_runtime_connection( + State(fixture.api.clone()), + AxumPath(binding.runtime_id.clone()), + )); + assert!(futures::poll!(&mut test).is_pending()); + observed(fixture.executor.drain()).await.unwrap(); + assert!(futures::poll!(&mut test).is_pending()); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!(request.contains("/ping"), "{request}"); + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let writer = fixture.store.clone(); + fixture + .executor + .dispatch(move || { + writer + .mark_api_token_used("missing-token", "2026-01-01T00:00:00Z") + .unwrap(); + }) + .unwrap(); + observed(waiting).await.unwrap(); + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + assert!( + futures::poll!(&mut test).is_pending(), + "post-ping binding fence must await DB without a synchronous mutex wait" + ); + runtime_progress_while_locked(&lock).await; + assert_eq!( + observed(fixture.host_executor.run(|| Ok::<_, Error>("control"))) + .await + .unwrap(), + "control" + ); + lock.release(); + busy.resume(); + let Json(result) = observed(test).await.unwrap(); + assert_eq!(result.binding_id, binding.binding_id); + fixture.executor.drain().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn saturated_host_rejects_runtime_key_revoke_without_mutating_trust() { + let fixture = Fixture::new().await; + fixture.api.worker_projection.shutdown_async().await; + observed(fixture.executor.drain()).await.unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let binding = install_remote_fixture( + &fixture, + "revoke-runtime", + &format!("http://{}", listener.local_addr().unwrap()), + true, + ); + fixture + .api + .runtime_binding_expectations + .write() + .unwrap() + .insert( + (WORKSPACE.into(), binding.runtime_id.clone()), + binding.clone(), + ); + let initial_registrations = fixture.api.runtime_subscription_broker.runtime_ids(); + // Sender Drop also releases each controlled job if a route assertion panics. + let mut releases: Vec> = Vec::new(); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + releases.push(release); + fixture + .host_executor + .dispatch(move || { + let _ = started.send(()); + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + observed(running).await.unwrap(); + let (release, permit) = mpsc::channel(); + releases.push(release); + fixture + .host_executor + .dispatch(move || { + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + let body = response_json( + observed(fixture.dispatch(Fixture::request( + "DELETE", + "/api/w/db-execution/runtimes/revoke-runtime/trust-key", + serde_json::json!({"expected_binding_id": binding.binding_id}), + ))) + .await + .unwrap(), + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + assert!(body.to_string().contains("db_capacity_exceeded"), "{body}"); + assert_eq!( + fixture + .store + .get_workspace_runtime_binding(WORKSPACE, &binding.runtime_id) + .unwrap() + .unwrap(), + binding + ); + assert!( + fixture + .store + .list_workspace_runtime_binding_audit(WORKSPACE, &binding.runtime_id, 10) + .unwrap() + .is_empty() + ); + assert_eq!( + fixture.api.runtime_subscription_broker.runtime_ids(), + initial_registrations + ); + assert_eq!( + fixture + .api + .runtime_binding_expectations + .read() + .unwrap() + .get(&(WORKSPACE.into(), binding.runtime_id.clone())), + Some(&binding) + ); + drop(releases); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_runtime_key_revoke_commits_audit_and_removes_broker_registration() { + let fixture = Fixture::new().await; + // Stop unrelated projection writes before observing this mutation's SQLite + // wait. Keep the original runtime alive through actual workflow cleanup. + fixture.api.worker_projection.shutdown_async().await; + observed(fixture.executor.drain()).await.unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let binding = install_remote_fixture( + &fixture, + "revoke-runtime", + &format!("http://{}", listener.local_addr().unwrap()), + true, + ); + fixture + .api + .runtime_binding_expectations + .write() + .unwrap() + .insert( + (WORKSPACE.into(), binding.runtime_id.clone()), + binding.clone(), + ); + let mut config = remote_runtime_config_from_binding(&binding).unwrap(); + config.workspace_authorization = Some(WorkspaceRuntimeAuthorization::new( + fixture.store.clone(), + fixture.api.signing_identities.clone(), + "http://localhost:8787", + binding.clone(), + )); + fixture + .api + .runtime_subscription_broker + .register_remote_runtime(config); + let mut subscription = fixture + .api + .runtime_subscription_broker + .subscribe( + &binding.runtime_id, + protocol::subscription::EventSubscriptionSelector::RuntimeWorkers, + ) + .unwrap(); + // Broker registration starts an independent cold signing DB job. Observe + // its actual handshake before occupying SQL for revoke: otherwise this + // fixture races that producer against HTTP auth in its 1+1 DB budget. + // Keep the handshake pending so downstream shutdown is still exercised. + let (mut upstream, _) = observed(listener.accept()).await.unwrap(); + let handshake = observed(async { + use tokio::io::AsyncReadExt; + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + let mut chunk = [0; 1024]; + let count = upstream.read(&mut chunk).await.unwrap(); + assert_ne!(count, 0, "broker closed before sending its handshake"); + bytes.extend_from_slice(&chunk[..count]); + assert!(bytes.len() <= 8192, "unexpectedly large handshake"); + } + String::from_utf8(bytes).unwrap() + }) + .await; + assert!(handshake.starts_with("GET /v1/protocol/ws ")); + assert!( + handshake + .to_ascii_lowercase() + .contains("authorization: bearer ") + ); + observed(fixture.executor.drain()).await.unwrap(); + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let server = fixture.server.clone(); + let request = Fixture::request( + "DELETE", + "/api/w/db-execution/runtimes/revoke-runtime/trust-key", + serde_json::json!({"expected_binding_id": binding.binding_id}), + ); + let mut revoke = + tokio::spawn(async move { dispatch_workspace_request(State(server), request).await }); + tokio::select! { + reached = observed(waiting) => reached.unwrap(), // Actual authenticated BEGIN IMMEDIATE. + response = &mut revoke => { + let response = response.unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 65536).await.unwrap(); + panic!("revoke returned before its SQLite wait: {status}: {}", String::from_utf8_lossy(&body)); + } + } + runtime_progress_while_locked(&lock).await; + revoke.abort(); + assert!(revoke.await.unwrap_err().is_cancelled()); + let admission = fixture.server.admission_for_workspace(WORKSPACE).await; + let mut deletion = Box::pin(admission.lock_deletion()); + assert!( + futures::poll!(&mut deletion).is_pending(), + "cancelled HTTP must retain actual revoke ownership until publication" + ); + assert!( + fixture + .api + .runtime_subscription_broker + .runtime_ids() + .contains(&binding.runtime_id) + ); + lock.release(); + busy.resume(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + let _deletion = observed(deletion).await; + let saved = fixture + .store + .get_workspace_runtime_binding(WORKSPACE, &binding.runtime_id) + .unwrap() + .unwrap(); + assert!(saved.revoked_at.is_some()); + assert_ne!(saved.binding_id, binding.binding_id); + let audit = fixture + .store + .list_workspace_runtime_binding_audit(WORKSPACE, &binding.runtime_id, 10) + .unwrap(); + assert_eq!( + audit.len(), + 1, + "cancelled mutation must commit one audit without retry" + ); + assert_eq!(audit[0].action, "revoked"); + assert_eq!(audit[0].actor_account_id, format!("account-{WORKSPACE}")); + assert_eq!(audit[0].binding_id, saved.binding_id); + assert_eq!(audit[0].at, saved.revoked_at.unwrap()); + assert!( + !fixture + .api + .runtime_subscription_broker + .runtime_ids() + .contains(&binding.runtime_id), + "committed revoke must unregister even after HTTP waiter cancellation" + ); + assert!( + !fixture + .api + .runtime_binding_expectations + .read() + .unwrap() + .contains_key(&(WORKSPACE.into(), binding.runtime_id.clone())) + ); + // Exercise downstream termination at the broker protocol boundary, not a + // product process or complete browser/Runtime E2E flow. + observed(async { + loop { + match subscription.recv().await { + Some(crate::runtime_subscription::BrokerSubscriptionEvent::Closed { + code, .. + }) => { + assert_eq!( + code, + protocol::subscription::SubscriptionTerminationCode::ServerShutdown + ); + break; + } + Some(crate::runtime_subscription::BrokerSubscriptionEvent::Disconnected { + .. + }) => {} + event => panic!("expected revoke shutdown, got {event:?}"), + } + } + }) + .await; +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_runtime_metadata_mutation_keeps_commit_and_registration_publication_owned() { + let fixture = Fixture::new().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + axum::serve( + listener, + Router::new().fallback(|| async { StatusCode::SERVICE_UNAVAILABLE }), + ) + .await + .unwrap(); + }); + install_remote_fixture(&fixture, "metadata-runtime", &endpoint, true); + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut update = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + update_remote_runtime_metadata_owned( + fixture.api.clone(), + WORKSPACE.into(), + "metadata-runtime".into(), + "updated metadata".into(), + endpoint, + ), + )); + assert!(futures::poll!(&mut update).is_pending()); + observed(waiting).await.unwrap(); // The real metadata SQL mutation is waiting. + runtime_progress_while_locked(&lock).await; + drop(update); + assert_eq!( + leases.load(Ordering::SeqCst), + 2, + "actual Host workflow and its accepted SQL both retain authority" + ); + lock.release(); + busy.resume(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + let saved = fixture + .store + .get_workspace_runtime_binding(WORKSPACE, "metadata-runtime") + .unwrap() + .unwrap(); + assert_eq!(saved.display_name, "updated metadata"); + let published = fixture + .api + .runtime_binding_expectations + .read() + .unwrap() + .get(&(WORKSPACE.into(), "metadata-runtime".into())) + .cloned() + .unwrap(); + assert_eq!( + published, saved, + "SQL completion must publish even after waiter cancellation" + ); + fixture + .api + .runtime_subscription_broker + .unregister_runtime("metadata-runtime"); + fixture.api.worker_projection.shutdown_async().await; + fixture.executor.drain().await.unwrap(); + server.abort(); + let _ = server.await; +} + +#[tokio::test(flavor = "current_thread")] +async fn runtime_metadata_reads_wait_off_tokio_and_db_and_retain_cancelled_host_ownership() { + for read in ["resources", "launch_options"] { + let fixture = Fixture::new().await; + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "summary-runtime", + &format!("http://{}", reply.address), + true, + ); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let api = fixture.api.clone(); + let task = tokio::spawn(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + async move { + if read == "resources" { + workspace_runtime_resources_response(&api, WORKSPACE) + .await + .map(|_| ()) + } else { + get_worker_launch_options(State(api)).await.map(|_| ()) + } + }, + )); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!(request.contains("/v1/runtime"), "{request}"); + assert!( + !reply.upstream_released.load(Ordering::SeqCst), + "upstream watchdog released before Tokio could observe the request" + ); + let read_store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || read_store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; // timer contract only + assert!(!reply.upstream_released.load(Ordering::SeqCst)); + assert!(!task.is_finished()); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!( + leases.load(Ordering::SeqCst), + 1, + "cancel released actual Runtime summary owner" + ); + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + fixture.executor.drain().await.unwrap(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn worker_discovery_sql_waits_off_tokio_and_keeps_admission_until_actual_cleanup() { + // Exercise the exact post-auth handler boundary: signed source verification + // has separate route coverage. Auth itself also awaits DB and would otherwise + // hide a handler's direct mutex acquisition until after the writer is released. + for cancel_waiter in [false, true] { + let fixture = Fixture::new().await; + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let writer = fixture.store.clone(); + fixture + .executor + .dispatch(move || { + writer + .mark_api_token_used("missing-token", "2026-01-01T00:00:00Z") + .unwrap(); + }) + .unwrap(); + observed(waiting).await.unwrap(); // Actual SQL writer holds shared connection mutex. + let query = || server_api::WorkspaceWorkerDiscoveryQuery { + limit: Some(1), + cursor: None, + query: Some("missing-peer".into()), + }; + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lifetime = Arc::new(CountJobLifetime(leases.clone())); + let mut discovery = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + discover_workspace_workers(fixture.api.clone(), WORKSPACE.into(), query(), true), + )); + assert!( + futures::poll!(&mut discovery).is_pending(), + "discovery must queue, not synchronously wait for the SQL mutex" + ); + assert_eq!(leases.load(Ordering::SeqCst), 1); + runtime_progress_while_locked(&lock).await; + assert_eq!( + observed(fixture.host_executor.run(|| Ok::<_, Error>("Host control"))) + .await + .unwrap(), + "Host control" + ); + assert!(futures::poll!(&mut discovery).is_pending()); + let rejected = + discover_workspace_workers(fixture.api.clone(), WORKSPACE.into(), query(), true) + .await + .unwrap_err() + .into_response(); + let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await; + assert!(body.to_string().contains("db_capacity_exceeded"), "{body}"); + let denied = + discover_workspace_workers(fixture.api.clone(), WORKSPACE.into(), query(), false) + .await + .unwrap_err() + .into_response(); + response_json(denied, StatusCode::FORBIDDEN).await; + // Failure cleanup never makes the progress assertions above true. + assert!(!lock.released.load(Ordering::SeqCst)); + if cancel_waiter { + drop(discovery); + assert_eq!( + leases.load(Ordering::SeqCst), + 1, + "cancel released accepted discovery authority" + ); + let rejected = + discover_workspace_workers(fixture.api.clone(), WORKSPACE.into(), query(), true) + .await + .unwrap_err() + .into_response(); + response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await; + lock.release(); + busy.resume(); + } else { + lock.release(); + busy.resume(); + let page = observed(discovery).await.unwrap(); + assert!( + page.workers.is_empty(), + "empty filtered projection must remain empty" + ); + assert!(page.next_cursor.is_none()); + } + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!( + leases.load(Ordering::SeqCst), + 0, + "actual discovery cleanup leaked its lease" + ); + fixture.host_executor.drain().await.unwrap(); + } +} diff --git a/crates/workspace-server/src/server_drive.rs b/crates/workspace-server/src/server_drive.rs index 57c79ef7f..33bb2e908 100644 --- a/crates/workspace-server/src/server_drive.rs +++ b/crates/workspace-server/src/server_drive.rs @@ -51,14 +51,13 @@ async fn blocking( mutation: bool, action: impl FnOnce() -> workspace_drive::Result + Send + 'static, ) -> DriveResult { - tokio::task::spawn_blocking(action) + crate::db_executor::DbExecutor::shared() + .run(move || Ok::<_, Error>(action())) .await - .map_err(|_| { - if mutation { - DriveApiError::new(DriveApiErrorCode::OutcomeUnknown) - } else { - failure(503) - } + .map_err(|error| match error { + Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503), + _ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown), + _ => failure(503), })? .map_err(|error| domain_error(error, mutation)) } diff --git a/crates/workspace-server/src/server_drive_grants.rs b/crates/workspace-server/src/server_drive_grants.rs index 9d85eaf67..8a82c51db 100644 --- a/crates/workspace-server/src/server_drive_grants.rs +++ b/crates/workspace-server/src/server_drive_grants.rs @@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into) -> DriveApiError { }) } fn mutation_error(error: Error) -> DriveApiError { - if matches!(error, Error::Sqlite(_) | Error::Store(_)) { + if matches!( + error, + Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown + ) { fail(DriveApiErrorCode::OutcomeUnknown) } else { pre_error(error) @@ -61,8 +64,9 @@ pub(super) async fn authorized_drive( // incomplete, or inconsistent with its transport identity. let headers = current_worker_contract_headers(context) .map_err(|_| fail(DriveApiErrorCode::Denied))?; - let worker = current_worker_identity(api, workspace_id, &headers) - .map_err(|_| fail(DriveApiErrorCode::Denied))?; + let worker = authenticate_worker_mutation_source_owned(api, workspace_id, &headers) + .await + .map_err(pre_error)?; if context.worker_source.as_ref().is_some_and(|source| { source.runtime_id != worker.runtime_id || source.worker_id != worker.worker_id }) { @@ -97,23 +101,27 @@ pub(super) async fn authorized_drive( }; // Reject missing/read-only grants before upload body reception. This check // is admission only: every later operation rechecks under its own lock. - tokio::task::spawn_blocking(move || { - drive.check_authorized(write)?; - Ok::<_, workspace_drive::Error>(drive) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? - .map_err(|error| { - fail(match error { - workspace_drive::Error::Denied => DriveApiErrorCode::Denied, - workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { - DriveApiErrorCode::Conflict - } - workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, - workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, - _ => DriveApiErrorCode::StorageUnavailable, + api.store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + drive.check_authorized(write)?; + Ok::<_, workspace_drive::Error>(drive) + })()) + }) + .await + .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? + .map_err(|error| { + fail(match error { + workspace_drive::Error::Denied => DriveApiErrorCode::Denied, + workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { + DriveApiErrorCode::Conflict + } + workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, + workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, + _ => DriveApiErrorCode::StorageUnavailable, + }) }) - }) } pub(super) async fn create_grant( @@ -132,9 +140,14 @@ pub(super) async fn create_grant( } } let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id); + let operation_api = api.clone(); let summary = api - .runtime - .worker(&worker) + .host_executors + .host() + .map_err(pre_error)? + .run(move || Ok::<_, Error>(operation_api.runtime.worker(&worker))) + .await + .map_err(pre_error)? .map_err(|_| fail(DriveApiErrorCode::NotFound))?; if summary.workspace.workspace_id.as_deref() != Some(workspace_id) { return Err(fail(DriveApiErrorCode::Denied)); @@ -152,9 +165,10 @@ pub(super) async fn create_grant( revoked_at: None, }; let store = api.store.clone(); - tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant)) + api.store + .db_executor() + .run(move || store.create_workspace_drive_grant(&grant)) .await - .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? .map_err(mutation_error) } @@ -175,13 +189,12 @@ pub(super) async fn revoke_grant( actor.account_id.clone(), ); let store = api.store.clone(); - tokio::task::spawn_blocking(move || { - store.revoke_workspace_drive_grant(&workspace, &grant, &actor) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? - .map_err(mutation_error)? - .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) + api.store + .db_executor() + .run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor)) + .await + .map_err(mutation_error)? + .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) } fn validate_grant_id(id: &str) -> DriveResult<()> { @@ -212,12 +225,14 @@ pub(super) async fn list_grants( } let workspace = workspace_id.to_owned(); let store = api.store.clone(); - let mut grants = tokio::task::spawn_blocking(move || { - store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? - .map_err(pre_error)?; + let mut grants = api + .store + .db_executor() + .run(move || { + store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) + }) + .await + .map_err(pre_error)?; let more = grants.len() > limit; grants.truncate(limit); let next_after = if more { diff --git a/crates/workspace-server/src/server_public_assignment_execution_tests.rs b/crates/workspace-server/src/server_public_assignment_execution_tests.rs new file mode 100644 index 000000000..b48394b97 --- /dev/null +++ b/crates/workspace-server/src/server_public_assignment_execution_tests.rs @@ -0,0 +1,120 @@ +//! Cross-lane composition regressions for T743's three new async DB callers. +//! Domain success/CAS/replay/rollback is covered by assignment_tracking_tests; +//! these exercise real generated handlers and their finite execution owners. +use super::*; + +struct Release(Option>); +impl Drop for Release { + fn drop(&mut self) { + self.0.take(); + } +} + +async fn queued_caller_keeps_owner( + method: &str, + suffix: &str, + body: serde_json::Value, + host: bool, +) { + let fixture = Fixture::new().await; + let executor = if host { + fixture.host_executor.clone() + } else { + fixture.executor.clone() + }; + let store = fixture.store.clone(); + let (entered, waiting) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut release = Release(Some(release)); + let watchdog = Arc::new(AtomicBool::new(false)); + let expired = watchdog.clone(); + // Occupy the actual owner and connection deterministically, not via sleeps + // or a request that could still be waiting in its authentication stage. + executor + .dispatch(move || { + store + .with_conn(|_| { + let _ = entered.send(()); + if permit.recv_timeout(WATCHDOG).is_err() { + expired.store(true, Ordering::SeqCst); + } + Ok(()) + }) + .unwrap(); + }) + .unwrap(); + observed(waiting).await.unwrap(); + let path = format!("/api/w/{WORKSPACE}/tickets/T-missing/{suffix}"); + let request = || Fixture::request(method, &path, body.clone()); + let (lifetime, accepted) = admission_observation(); + let mut queued = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture.inner_router().oneshot(request()), + )); + assert!(futures::poll!(&mut queued).is_pending()); + observed(accepted).await.unwrap(); + let control = tokio::spawn(async { "independent control" }); + assert_eq!(observed(control).await.unwrap(), "independent control"); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !watchdog.load(Ordering::SeqCst), + "synchronous caller stalled Tokio until the watchdog" + ); + drop(queued); + // Cancellation of the waiter cannot free the accepted queued SQL slot. + let rejected = observed(fixture.inner_router().oneshot(request())) + .await + .unwrap(); + let error = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await; + assert!( + error["message"].as_str().unwrap().contains("capacity"), + "{error}" + ); + let mut drained = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + release.0.take().unwrap().send(()).unwrap(); + observed(drained).await.unwrap(); + // The accepted request finishes its real lookup after cancellation; the + // normal domain error and reusable capacity survive without SQL resending. + let after = observed(fixture.inner_router().oneshot(request())) + .await + .unwrap(); + response_json(after, StatusCode::NOT_FOUND).await; + observed(executor.drain()).await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn assignments_page_uses_finite_db_owner_and_survives_cancel() { + queued_caller_keeps_owner( + "GET", + "assignment-tracking/assignments", + serde_json::json!({}), + false, + ) + .await; +} + +#[tokio::test(flavor = "current_thread")] +async fn diagnostics_page_uses_finite_db_owner_and_survives_cancel() { + queued_caller_keeps_owner( + "GET", + "assignment-diagnostics?assignment_id=missing", + serde_json::json!({}), + false, + ) + .await; +} + +#[tokio::test(flavor = "current_thread")] +async fn answer_uses_finite_host_owner_and_survives_cancel() { + queued_caller_keeps_owner( + "POST", + "assignment-answer", + serde_json::json!({ + "assignment_id":"missing", "expected_last_event_id":"missing", + "operation_key":"answer-probe", "body":"continuing input" + }), + true, + ) + .await; +} diff --git a/crates/workspace-server/src/server_residual_execution_tests.rs b/crates/workspace-server/src/server_residual_execution_tests.rs new file mode 100644 index 000000000..6cf3759fa --- /dev/null +++ b/crates/workspace-server/src/server_residual_execution_tests.rs @@ -0,0 +1,1381 @@ +use super::*; + +fn registry_record(worker: RuntimeWorkerRef) -> WorkerRegistryRecord { + WorkerRegistryRecord { + workspace_id: WORKSPACE.into(), + worker, + display_name: "current stored label".into(), + profile: Some("builtin:coder".into()), + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + } +} + +fn assigned_ticket(fixture: &Fixture) -> String { + let record = registry_record(RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + Uuid::now_v7().to_string(), + )); + fixture.store.upsert_worker_registry(&record).unwrap(); + let mut input = ticket::NewTicket::new("cancel this implementation"); + input.workflow_state = Some(TicketState::Open); + let ticket = browser_ticket_backend(&fixture.api) + .unwrap() + .create(input) + .unwrap(); + fixture + .store + .set_current_ticket_worker_assignment( + &TicketWorkerAssignmentRecord { + workspace_id: WORKSPACE.into(), + ticket_id: ticket.id.clone(), + assignment_id: "current-coder".into(), + worker: record.worker, + assigned_by: "fixture-user".into(), + assigned_at: "2026-01-01T00:00:00Z".into(), + }, + None, + "assign-event", + "assign-operation", + false, + ) + .unwrap(); + ticket.id +} + +async fn fixture() -> Fixture { + let fixture = Fixture::new().await; + fixture.api.worker_projection.shutdown_async().await; + observed(fixture.executor.drain()).await.unwrap(); + fixture +} + +fn cancel_request() -> serde_json::Value { + serde_json::json!({ + "operation_id": "cancel-operation", "assignment_id": "current-coder", "reason": "use the corrected design" + }) +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_ticket_implementation_keeps_assignment_commit_owned() { + let fixture = fixture().await; + let ticket = assigned_ticket(&fixture); + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + fixture.inner_router().oneshot(Fixture::request( + "POST", + &format!("/api/w/{WORKSPACE}/tickets/{ticket}/implementation-cancellations"), + cancel_request(), + )), + )); + assert!( + futures::poll!(&mut response).is_pending(), + "cancellation must admit an owned stage, not run SQL on Tokio" + ); + observed(waiting).await.unwrap(); // Actual cancellation transaction, after Runtime/session cleanup. + runtime_progress_while_locked(&lock).await; + drop(response); + assert_eq!( + leases.load(Ordering::SeqCst), + 1, + "actual mixed cancellation must keep its authority after the waiter drops" + ); + lock.release(); + busy.resume(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + assert!( + fixture + .store + .get_active_ticket_worker_assignment(WORKSPACE, &ticket) + .unwrap() + .is_none() + ); + assert!( + fixture + .store + .get_ticket_assignment_operation(WORKSPACE, "cancel-operation") + .unwrap() + .is_some() + ); + let detail = browser_ticket_detail(&fixture.api, &ticket).unwrap().0; + assert!(detail.current_worker.is_none()); + assert_eq!(detail.state.as_str(), TicketState::Open.as_str()); +} + +async fn saturated_host(fixture: &Fixture) -> Vec> { + let mut releases = Vec::new(); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel::<()>(); + releases.push(release); + fixture + .host_executor + .dispatch(move || { + let _ = started.send(()); + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + observed(running).await.unwrap(); + let (release, permit) = mpsc::channel::<()>(); + releases.push(release); + fixture + .host_executor + .dispatch(move || { + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + releases +} + +#[tokio::test(flavor = "current_thread")] +async fn cancellation_host_overload_leaves_the_coder_assignment_unchanged() { + let fixture = fixture().await; + let ticket = assigned_ticket(&fixture); + let before = fixture + .store + .get_active_ticket_worker_assignment(WORKSPACE, &ticket) + .unwrap() + .unwrap(); + let releases = saturated_host(&fixture).await; + let body = response_json( + observed(fixture.inner_router().oneshot(Fixture::request( + "POST", + &format!("/api/w/{WORKSPACE}/tickets/{ticket}/implementation-cancellations"), + cancel_request(), + ))) + .await + .unwrap(), + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + assert!(body.to_string().contains("db_capacity_exceeded"), "{body}"); + assert_eq!( + fixture + .store + .get_active_ticket_worker_assignment(WORKSPACE, &ticket) + .unwrap() + .unwrap(), + before + ); + assert!( + fixture + .store + .get_ticket_assignment_operation(WORKSPACE, "cancel-operation") + .unwrap() + .is_none() + ); + drop(releases); + observed(fixture.host_executor.drain()).await.unwrap(); +} + +fn seed_unrepaired_mr(fixture: &Fixture) { + let ticket = browser_ticket_backend(&fixture.api) + .unwrap() + .create(ticket::NewTicket::new("repair source selector")) + .unwrap(); + fixture + .store + .upsert_repository(&crate::store::RepositoryRecord { + workspace_id: WORKSPACE.into(), + repository_id: "fixture-repo".into(), + repository_key: "fixture-repo".into(), + kind: "git".into(), + provider: Some("git".into()), + source: server_api::RepositorySource { + kind: server_api::RepositorySourceKind::LocalPath, + uri: fixture + ._dir + .path() + .join("unused-repository") + .to_string_lossy() + .into_owned(), + }, + default_ref: Some("develop".into()), + source_fingerprint: "sha256:fixture".into(), + observed_status: server_api::RepositoryObservedStatus::Unverified, + observed_at: None, + created_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + }) + .unwrap(); + // Persist the external unrepaired-selector shape. No active assignment: + // the real handler must reach its assignment read, then reject without provider dispatch. + let conn = Connection::open(&fixture.api.config.database_path).unwrap(); + conn.execute("INSERT INTO merge_requests VALUES (?1,'repair-fixture','fixture-repo','open',NULL,'develop',?2,?2)", rusqlite::params![WORKSPACE, "2026-01-01T00:00:00Z"]).unwrap(); + conn.execute("INSERT INTO merge_request_ticket_relations VALUES (?1,'repair-fixture',?2,'implements',?3)", rusqlite::params![WORKSPACE, ticket.id, "2026-01-01T00:00:00Z"]).unwrap(); +} + +fn repair_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + "cookie", + "db_execution_session=fixture-browser-token" + .parse() + .unwrap(), + ); + headers +} + +fn repair_input() -> Json { + Json(server_api::RepairMergeRequestSelectorRequest { + selector_from: "work/fixed-selector".into(), + reason: "repair the imported selector".into(), + explicit_confirmation: true, + }) +} + +#[tokio::test(flavor = "current_thread")] +async fn authenticated_selector_repair_waits_off_tokio_and_keeps_cancelled_host_ownership() { + let fixture = fixture().await; + seed_unrepaired_mr(&fixture); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut repair = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + scoped_repair_merge_request_selector( + State(fixture.api.clone()), + repair_headers(), + AxumPath((WORKSPACE.into(), "repair-fixture".into())), + repair_input(), + ), + )); + assert!(futures::poll!(&mut repair).is_pending()); // Browser authentication DB stage only. + observed(fixture.executor.drain()).await.unwrap(); + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let writer = fixture.store.clone(); + fixture + .executor + .dispatch(move || { + writer + .mark_api_token_used("missing", "2026-01-01T00:00:00Z") + .unwrap(); + }) + .unwrap(); + observed(waiting).await.unwrap(); + assert!( + futures::poll!(&mut repair).is_pending(), + "post-auth selector repair must not wait on the connection mutex on Tokio" + ); + runtime_progress_while_locked(&lock).await; + drop(repair); + assert_eq!( + leases.load(Ordering::SeqCst), + 1, + "actual repair stage must retain authority after cancellation" + ); + lock.release(); + busy.resume(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + let mr = merge_request_store(&fixture.api, WORKSPACE) + .unwrap() + .get_by_id(WORKSPACE, "repair-fixture") + .unwrap(); + assert!(mr.selector_from.is_none()); + assert!( + merge_request_store(&fixture.api, WORKSPACE) + .unwrap() + .thread_page_by_id(WORKSPACE, "repair-fixture", None, 10) + .unwrap() + .is_empty() + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn selector_repair_host_overload_preserves_confirmation_and_no_side_effects() { + let fixture = fixture().await; + seed_unrepaired_mr(&fixture); + let releases = saturated_host(&fixture).await; + let error = scoped_repair_merge_request_selector( + State(fixture.api.clone()), + repair_headers(), + AxumPath((WORKSPACE.into(), "repair-fixture".into())), + repair_input(), + ) + .await + .unwrap_err(); + let body = response_json(error.into_response(), StatusCode::SERVICE_UNAVAILABLE).await; + assert!(body.to_string().contains("db_capacity_exceeded"), "{body}"); + let mut input = repair_input(); + input.0.explicit_confirmation = false; + let error = scoped_repair_merge_request_selector( + State(fixture.api.clone()), + repair_headers(), + AxumPath((WORKSPACE.into(), "repair-fixture".into())), + input, + ) + .await + .unwrap_err(); + assert!( + !matches!(error.error, Error::DbCapacityExceeded), + "browser confirmation denial must precede Host admission" + ); + let mr = merge_request_store(&fixture.api, WORKSPACE) + .unwrap() + .get_by_id(WORKSPACE, "repair-fixture") + .unwrap(); + assert!(mr.selector_from.is_none()); + drop(releases); + observed(fixture.host_executor.drain()).await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn post_restore_projection_waits_off_tokio_and_preserves_historical_results() { + let fixture = fixture().await; + let record = registry_record(RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + Uuid::now_v7().to_string(), + )); + fixture.store.upsert_worker_registry(&record).unwrap(); + let mut historical = worker_summary_from_registry(&record); + historical.label = "historical label".into(); + historical.state = "paused".into(); + let result = || crate::hosts::InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::Rejected, + worker: Some(historical.clone()), + diagnostics: Vec::new(), + }; + let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let writer = fixture.store.clone(); + fixture + .executor + .dispatch(move || { + writer + .mark_api_token_used("missing", "2026-01-01T00:00:00Z") + .unwrap(); + }) + .unwrap(); + observed(waiting).await.unwrap(); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut projection = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + project_worker_restore_response_owned( + fixture.api.clone(), + record.worker.runtime_id.clone(), + record.worker.worker_id.clone(), + result(), + ), + )); + assert!( + futures::poll!(&mut projection).is_pending(), + "stored projection must await a whole DB stage after Host restore returns" + ); + runtime_progress_while_locked(&lock).await; + assert_eq!( + observed(fixture.host_executor.run(|| Ok::<_, Error>("control"))) + .await + .unwrap(), + "control" + ); + let error = project_worker_restore_response_owned( + fixture.api.clone(), + record.worker.runtime_id.clone(), + record.worker.worker_id.clone(), + result(), + ) + .await + .unwrap_err(); + response_json(error.into_response(), StatusCode::SERVICE_UNAVAILABLE).await; + let Json(no_worker) = project_worker_restore_response_owned( + fixture.api.clone(), + record.worker.runtime_id.clone(), + record.worker.worker_id.clone(), + crate::hosts::InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::Rejected, + worker: None, + diagnostics: Vec::new(), + }, + ) + .await + .unwrap(); + assert_eq!( + no_worker.result.state, + server_api::WorkerRestoreState::Rejected + ); + drop(projection); + assert_eq!(leases.load(Ordering::SeqCst), 1); + lock.release(); + busy.resume(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + let Json(response) = project_worker_restore_response_owned( + fixture.api.clone(), + record.worker.runtime_id.clone(), + record.worker.worker_id.clone(), + result(), + ) + .await + .unwrap(); + assert_eq!( + response.result.state, + server_api::WorkerRestoreState::Rejected + ); + let worker = response.result.worker.unwrap(); + assert_eq!(worker.label, record.display_name); + assert_eq!( + worker.state, "paused", + "a historical receipt is not a new Runtime observation" + ); + assert_eq!( + fixture + .store + .get_worker_registry(WORKSPACE, &record.worker) + .unwrap() + .unwrap(), + record, + "response projection must not write a new observation" + ); + observed(fixture.executor.drain()).await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn ticket_source_ref_reads_and_ready_publication_leave_sql_and_tokio_free() { + for operation in ["detail", "query", "ready"] { + let repository_dir = tempfile::tempdir().unwrap(); + let source = server_api::RepositorySource { + kind: server_api::RepositorySourceKind::LocalPath, + uri: repository_dir.path().to_string_lossy().into_owned(), + }; + let fixture = Fixture::with_repositories(vec![ConfiguredRepository { + id: "fixture-repo".into(), + repository_key: "fixture-repo".into(), + provider: "git".into(), + source_fingerprint: crate::repository_source::repository_source_fingerprint(&source), + source, + observed_status: server_api::RepositoryObservedStatus::Unverified, + observed_at: None, + path: Some(repository_dir.path().into()), + default_selector: Some("develop".into()), + }]) + .await; + fixture.api.worker_projection.shutdown_async().await; + observed(fixture.executor.drain()).await.unwrap(); + let mut reply = HeldRuntimeReply::for_path("/v1/repository-refs/observe"); + // Narrow Runtime HTTP/provider placement fixture, not a signed authority + // or complete Git/Runtime E2E test. Existing signed routes test that fence. + install_remote_fixture( + &fixture, + "ref-runtime", + &format!("http://{}", reply.address), + false, + ); + let worker = RuntimeWorkerRef::new("ref-runtime", Uuid::now_v7().to_string()); + fixture + .store + .upsert_worker_registry(®istry_record(worker.clone())) + .unwrap(); + let mut input = ticket::NewTicket::new("source ref projection"); + input.workflow_state = Some(TicketState::Open); + let ticket = browser_ticket_backend(&fixture.api) + .unwrap() + .create(input) + .unwrap(); + fixture + .store + .set_current_ticket_worker_assignment( + &TicketWorkerAssignmentRecord { + workspace_id: WORKSPACE.into(), + ticket_id: ticket.id.clone(), + assignment_id: "ref-assignment".into(), + worker: worker.clone(), + assigned_by: "fixture-user".into(), + assigned_at: "2026-01-01T00:00:00Z".into(), + }, + None, + "ref-event", + "ref-operation", + false, + ) + .unwrap(); + merge_request_store(&fixture.api, WORKSPACE) + .unwrap() + .open_merge_request(merge_request::OpenMergeRequest { + merge_request_id: "ref-mr".into(), + ticket_id: ticket.id.clone(), + repository_id: "fixture-repo".into(), + selector_from: "work/source".into(), + selector_to: "develop".into(), + summary: String::new(), + auth: merge_request::MergeRequestAuth { + workspace_id: WORKSPACE.into(), + repository_id: "fixture-repo".into(), + runtime_id: worker.runtime_id, + worker_id: worker.worker_id, + assignment_id: "ref-assignment".into(), + }, + now: Utc::now(), + }) + .unwrap(); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let api = fixture.api.clone(); + let ticket_id = ticket.id.clone(); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + async move { + match operation { + "detail" => scoped_get_ticket( + State(api), + AxumPath(ScopedRecordPath { + workspace_id: WORKSPACE.into(), + id: ticket_id, + }), + ) + .await + .map(|_| ()), + "query" => scoped_query_tickets( + State(api), + AxumPath(ScopedWorkspacePath { + workspace_id: WORKSPACE.into(), + }), + Json(TicketQueryRequest::default()), + ) + .await + .map(|_| ()), + _ => scoped_mark_ticket_ready_from_browser( + State(api), + HeaderMap::new(), + AxumPath(ScopedRecordPath { + workspace_id: WORKSPACE.into(), + id: ticket_id, + }), + Json(InternalTicketMarkReadyRequest { + operation_key: "ready-operation".into(), + reason: None, + intake_summary: None, + }), + ) + .await + .map(|_| ()), + } + }, + )); + assert!(futures::poll!(&mut response).is_pending()); + // Ready reaches the source POST only after mutation and best-effort + // notification. Bound this multi-stage setup with the fixture watchdog, + // not the shorter budget used below to prove timer/independent control + // progress while the actual provider request is held. + let request = tokio::select! { + request = tokio::time::timeout(WATCHDOG, reply.entered.take().unwrap()) => request.expect("source observation setup did not reach the provider").unwrap(), + result = &mut response => panic!("{operation} returned before the held source observation: {result:?}"), + }; + assert!( + request.contains("/v1/repository-refs/observe"), + "{operation}: {request}" + ); + let store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !reply.upstream_released.load(Ordering::SeqCst), + "watchdog release is not provider progress" + ); + if operation == "ready" { + let saved = browser_ticket_backend(&fixture.api) + .unwrap() + .show(TicketIdOrSlug::Id(ticket.id)) + .unwrap(); + assert_eq!( + saved.meta.readiness.as_deref(), + Some("ready"), + "Ready mutation must be durable before the held detail source observation" + ); + assert_eq!( + saved + .events + .iter() + .filter( + |event| event.attributes.get("operation_key").map(String::as_str) + == Some("ready-operation") + ) + .count(), + 1 + ); + assert_eq!( + saved.meta.workflow_state, + TicketState::Open, + "Ready must commit before detail publication" + ); + } + drop(response); + assert_eq!( + leases.load(Ordering::SeqCst), + 1, + "{operation}: cancelled provider stage lost actual ownership" + ); + fixture.host_executor.dispatch(|| {}).unwrap(); // Remaining finite queued slot. + let error = scoped_get_ticket( + State(fixture.api.clone()), + AxumPath(ScopedRecordPath { + workspace_id: WORKSPACE.into(), + id: "unused".into(), + }), + ) + .await + .unwrap_err(); + response_json(error.into_response(), StatusCode::SERVICE_UNAVAILABLE).await; + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn worker_control_source_lookup_waits_off_tokio_and_retains_finite_host_ownership() { + let fixture = fixture().await; + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "source-runtime", + &format!("http://{}", reply.address), + false, + ); + let source = RuntimeWorkerRef::new("source-runtime", Uuid::now_v7().to_string()); + let mut headers = HeaderMap::new(); + headers.insert("x-yoi-runtime-id", source.runtime_id.parse().unwrap()); + headers.insert("x-yoi-worker-id", source.worker_id.parse().unwrap()); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let request = || { + Json(server_api::RuntimeWorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }) + }; + let path = || { + AxumPath(ScopedRuntimeWorkerPath { + workspace_id: WORKSPACE.into(), + worker: source.clone(), + }) + }; + let mut control = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + cancel_known_worker( + State(fixture.api.clone()), + path(), + headers.clone(), + request(), + ), + )); + assert!( + futures::poll!(&mut control).is_pending(), + "control source lookup must not synchronously join Runtime HTTP on Tokio" + ); + let request_line = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!(request_line.contains("/v1/workers/"), "{request_line}"); + let store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!(!reply.upstream_released.load(Ordering::SeqCst)); + drop(control); + assert_eq!(leases.load(Ordering::SeqCst), 1); + fixture.host_executor.dispatch(|| {}).unwrap(); + let error = cancel_known_worker(State(fixture.api.clone()), path(), headers, request()) + .await + .unwrap_err(); + let body = response_json(error.into_response(), StatusCode::SERVICE_UNAVAILABLE).await; + assert!( + body.to_string().contains("db_capacity_exceeded"), + "Host overload is availability, not identity denial: {body}" + ); + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); +} + +#[tokio::test(flavor = "current_thread")] +async fn ticket_tracking_and_case_authority_runtime_lookup_leave_db_and_tokio_free() { + for operation in ["tracking", "case"] { + let fixture = fixture().await; + let ticket = browser_ticket_backend(&fixture.api) + .unwrap() + .create(ticket::NewTicket::new("decision authority")) + .unwrap(); + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "orchestrator-runtime", + &format!("http://{}", reply.address), + false, + ); + let memory = fixture + .store + .get_workspace_memory_settings(WORKSPACE) + .unwrap(); + let reservation = fixture + .store + .reserve_worker_create( + WORKSPACE, + "orchestrator-runtime", + "orch-create", + "orch-create-fingerprint", + Some(crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY), + &memory, + ) + .unwrap(); + let source = + RuntimeWorkerRef::new("orchestrator-runtime", reservation.worker_id.to_string()); + fixture + .store + .upsert_worker_registry(®istry_record(source.clone())) + .unwrap(); + fixture + .store + .complete_worker_create_reservation(WORKSPACE, reservation.worker_id) + .unwrap(); + let context = server_api::ServerRequestContext { + actor: None, + worker_source: None, + runtime_source: Some(server_api::ServerRuntimeSource { + authentication_binding_id: "verified-fixture-binding".into(), + runtime_id: source.runtime_id.clone(), + worker_id: Some(source.worker_id.clone()), + }), + origin: None, + transport_headers: Vec::new(), + }; + let backend = browser_ticket_backend(&fixture.api).unwrap(); + let before = backend.show(TicketIdOrSlug::Id(ticket.id.clone())).unwrap(); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let api = fixture.api.clone(); + let ticket_id = ticket.id.clone(); + let mut decision = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + async { + if operation == "tracking" { + scoped_ticket_assignment_tracking_update_with_context( + State(api), + HeaderMap::new(), + &context, + AxumPath((WORKSPACE.into(), ticket_id)), + Json(ticket::TicketAssignmentTrackingUpdate { + assignment_id: "no-delegation".into(), + expected_last_event_id: "baseline".into(), + operation_key: "track-operation".into(), + tracking: ticket::TicketAssignmentTracking { + status: ticket::TicketAssignmentTrackingStatus::Inprogress, + waiting_reason: None, + progress: None, + next_action: None, + result: None, + }, + }), + ) + .await + .map(|_| ()) + } else { + apply_ticket_case_update( + State(api), + Some(source), + None, + AxumPath((WORKSPACE.into(), ticket_id)), + Json(ticket::TicketCaseUpdate { + expected_content_digest: ticket::ticket_content_digest(&before), + expected_state: before.meta.workflow_state, + expected_closure_operation_key: None, + operation_key: "case-operation".into(), + action: ticket::TicketCaseAction::Close { + reason: ticket::TicketClosureReason::Completed, + }, + explanation: "check current authority".into(), + references: Vec::new(), + author: None, + }), + ) + .await + .map(|_| ()) + } + }, + )); + assert!(futures::poll!(&mut decision).is_pending()); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!(request.contains("/v1/workers/"), "{operation}: {request}"); + let store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!(!reply.upstream_released.load(Ordering::SeqCst)); + drop(decision); + assert_eq!(leases.load(Ordering::SeqCst), 1); + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + assert_eq!( + backend.show(TicketIdOrSlug::Id(ticket.id)).unwrap(), + before, + "failed live authority must not make a decision" + ); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_feature_shutdown_keeps_sql_close_owned_without_blocking_control() { + let fixture = fixture().await; + let (database, _) = fixture.feature_database(); + let mutation = database.clone(); + let (entered, waiting) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let released = Arc::new(AtomicBool::new(false)); + let flag = released.clone(); + let thread = std::thread::spawn(move || { + mutation + .with_connection(|conn| { + conn.execute_batch("BEGIN IMMEDIATE; CREATE TABLE close_owner_probe(id INTEGER)")?; + let _ = entered.send(()); + let _ = permit.recv_timeout(WATCHDOG); + flag.store(true, Ordering::SeqCst); + conn.execute_batch("ROLLBACK")?; + Ok(()) + }) + .unwrap(); + }); + let mut lock = WriteLock { + release: Some(release), + thread: Some(thread), + released, + }; + observed(waiting).await.unwrap(); // actual SQL transaction owns the Feature connection + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut close = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + shutdown_workspace_feature_storage(&fixture.api), + )); + assert!( + futures::poll!(&mut close).is_pending(), + "Feature connection close must not wait on Tokio" + ); + runtime_progress_while_locked(&lock).await; + assert_eq!( + observed(fixture.host_executor.run(|| Ok::<_, Error>("control"))) + .await + .unwrap(), + "control" + ); + fixture.executor.dispatch(|| {}).unwrap(); + assert!(matches!( + shutdown_workspace_feature_storage(&fixture.api) + .await + .unwrap_err(), + Error::DbCapacityExceeded + )); + drop(close); + assert_eq!(leases.load(Ordering::SeqCst), 1); + lock.release(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + assert!( + database.with_connection(|_| Ok(())).is_err(), + "accepted shutdown must fence later use" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn session_workdir_and_upload_callers_keep_provider_waits_off_sql_and_tokio() { + use server_api::ServerApi; + for operation in ["session", "history", "attachment", "workdirs", "upload"] { + let fixture = fixture().await; + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "caller-runtime", + &format!("http://{}", reply.address), + false, + ); + let worker_id = Uuid::now_v7().to_string(); + fixture + .store + .upsert_worker_registry(®istry_record(RuntimeWorkerRef::new( + "caller-runtime", + &worker_id, + ))) + .unwrap(); + let actor = RequestActor { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + auth_method: ActorAuthMethod::BrowserSession, + }; + fixture.api.attachment_upload_grants.lock().unwrap().insert( + "caller-upload".into(), + AttachmentUploadGrant { + workspace_id: WORKSPACE.into(), + runtime_id: "caller-runtime".into(), + worker_id: worker_id.clone(), + account_id: actor.account_id.clone(), + file_name: "notes.txt".into(), + media_type: "text/plain".into(), + expires_at_ms: u64::MAX, + state: AttachmentUploadGrantState::Pending, + }, + ); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let api = fixture.api.clone(); + let task = tokio::spawn(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + async move { + let service = ServerApiContractService::Workspace(api.clone()); + match operation { + "session" => { + let _ = service + .worker_session(WORKSPACE.into(), "caller-runtime".into(), worker_id) + .await; + } + "history" => { + let _ = service + .worker_session_history( + WORKSPACE.into(), + "caller-runtime".into(), + worker_id, + server_api::WorkspaceWorkerSessionHistoryQuery { + cursor: None, + limit: Some(1), + }, + ) + .await; + } + "attachment" => { + let _ = scoped_worker_session_attachment( + State(api), + AxumPath(( + WORKSPACE.into(), + "caller-runtime".into(), + worker_id, + "session".into(), + "attachment".into(), + )), + Extension(actor), + ) + .await; + } + "workdirs" => { + let _ = scoped_list_working_directories( + State(api), + AxumPath(ScopedWorkspacePath { + workspace_id: WORKSPACE.into(), + }), + ) + .await; + } + "upload" => { + let _ = upload_runtime_worker_file_contract( + api, + actor, + ScopedAttachmentUploadPath { + workspace_id: WORKSPACE.into(), + runtime_id: "caller-runtime".into(), + worker_id, + upload_id: "caller-upload".into(), + }, + b"notes", + ) + .await; + } + _ => unreachable!(), + } + }, + )); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!(request.contains("/v1/"), "{operation}: {request}"); + assert!( + !reply.upstream_released.load(Ordering::SeqCst), + "{operation}: watchdog release is not executor progress" + ); + let store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + let mut queued = Box::pin(fixture.host_executor.run(|| Ok::<_, Error>(()))); + assert!(futures::poll!(&mut queued).is_pending()); + let error = scoped_list_working_directories( + State(fixture.api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: WORKSPACE.into(), + }), + ) + .await + .unwrap_err(); + response_json(error.into_response(), StatusCode::SERVICE_UNAVAILABLE).await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!( + leases.load(Ordering::SeqCst), + 1, + "{operation}: lost provider owner" + ); + reply.finish(); + observed(queued).await.unwrap(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!( + leases.load(Ordering::SeqCst), + 0, + "{operation}: retained owner after completion" + ); + if operation == "upload" { + assert!( + matches!( + fixture + .api + .attachment_upload_grants + .lock() + .unwrap() + .get("caller-upload") + .unwrap() + .state, + AttachmentUploadGrantState::Pending + ), + "failed Runtime lookup must finish the accepted grant claim even with a cancelled waiter" + ); + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn repository_create_keeps_cancelled_sql_commit_owned_and_replays_one_intent() { + let fixture = fixture().await; + let (mut busy, waiting) = BusyObservation::install_store(fixture.api.config_store.clone()); + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let request = || { + Fixture::request( + "POST", + &format!("/api/w/{WORKSPACE}/repositories"), + serde_json::json!({ "repository_key": "owned-repository", "source": "https://example.invalid/repo.git", "default_ref": "main" }), + ) + }; + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + fixture.inner_router().oneshot(request()), + )); + tokio::select! { + result = &mut response => panic!("repository must wait for its write transaction: {:?}", result.unwrap().status()), + reached = observed(waiting) => reached.unwrap(), + } + runtime_progress_while_locked(&lock).await; + assert_eq!( + observed(fixture.host_executor.run(|| Ok::<_, Error>("control"))) + .await + .unwrap(), + "control" + ); + drop(response); + assert_eq!(leases.load(Ordering::SeqCst), 1); + lock.release(); + busy.resume(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + let rows = fixture.store.list_repositories(WORKSPACE).unwrap(); + assert_eq!( + rows.iter() + .filter(|record| record.repository_key == "owned-repository") + .count(), + 1 + ); + let replay = observed(fixture.inner_router().oneshot(request())) + .await + .unwrap(); + let body = response_json(replay, StatusCode::OK).await; + assert_eq!(body["replayed"], true); + assert_eq!( + fixture.store.list_repositories(WORKSPACE).unwrap().len(), + rows.len() + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn drive_and_config_preflight_keep_live_authority_owned_and_report_overload_as_availability() +{ + async fn preflight( + api: &WorkspaceApi, + context: &server_api::ServerRequestContext, + operation: &str, + ) -> u16 { + match operation { + "config" => { + workspace_config::identity(api, context, WORKSPACE) + .await + .err() + .unwrap() + .status + } + "drive" => api_macros::HttpError::status_code( + &drive_grants::authorized_drive(api, context, WORKSPACE, false) + .await + .err() + .unwrap(), + ), + "grant" => api_macros::HttpError::status_code( + &drive_grants::create_grant( + api, + context, + WORKSPACE, + server_api::DriveGrantCreateRequest { + runtime_id: "preflight-runtime".into(), + worker_id: "target-worker".into(), + access: server_api::DriveAccess::ReadOnly, + }, + ) + .await + .err() + .unwrap(), + ), + _ => unreachable!(), + } + } + for operation in ["config", "drive", "grant"] { + let fixture = fixture().await; + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "preflight-runtime", + &format!("http://{}", reply.address), + false, + ); + let context = server_api::ServerRequestContext { + actor: if operation == "grant" { + Some(RequestActor { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + auth_method: ActorAuthMethod::BrowserSession, + }) + } else { + None + }, + worker_source: None, + runtime_source: if operation == "grant" { + None + } else { + Some(server_api::ServerRuntimeSource { + authentication_binding_id: "fixture-binding".into(), + runtime_id: "preflight-runtime".into(), + worker_id: Some("source-worker".into()), + }) + }, + origin: None, + transport_headers: Vec::new(), + }; + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let api = fixture.api.clone(); + let request_context = context.clone(); + let task = tokio::spawn(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + async move { preflight(&api, &request_context, operation).await }, + )); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!(request.contains("/v1/workers/"), "{operation}: {request}"); + let store = fixture.store.clone(); + observed( + fixture + .executor + .run(move || store.get_workspace_memory_settings(WORKSPACE)), + ) + .await + .unwrap(); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + assert!(!reply.upstream_released.load(Ordering::SeqCst)); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(leases.load(Ordering::SeqCst), 1); + fixture.host_executor.dispatch(|| {}).unwrap(); + assert_eq!( + observed(preflight(&fixture.api, &context, operation)).await, + 503, + "{operation}: overload must not be authority denial" + ); + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn ui_config_commit_publishes_after_commit_without_holding_the_sql_lane() { + let fixture = fixture().await; + let current = fixture + .api + .config_store + .load_workspace_config(WORKSPACE) + .unwrap() + .unwrap(); + let mut reply = HeldRuntimeReply::new(); + install_remote_fixture( + &fixture, + "prompt-runtime", + &format!("http://{}", reply.address), + false, + ); + let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut commit = Box::pin(crate::db_executor::with_operation_lifetime( + Arc::new(CountJobLifetime(leases.clone())), + scoped_commit_workspace_config_tree( + State(fixture.api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: WORKSPACE.into(), + }), + Json(server_api::ConfigCommitRequest { + base_digest: current.snapshot.digest.clone(), + entrypoints: current + .contract + .entrypoints + .iter() + .map(ToString::to_string) + .collect(), + changes: vec![server_api::ConfigTreeChange::Create { + path: "notes/owned-publication.txt".into(), + content_type: server_api::ConfigContentType::Text, + content: "committed before prompt publication".into(), + }], + }), + ), + )); + assert!(futures::poll!(&mut commit).is_pending()); + let request = observed(reply.entered.take().unwrap()).await.unwrap(); + assert!( + request.starts_with("POST "), + "prompt publication: {request}" + ); + let config_store = fixture.api.config_store.clone(); + let state = observed( + fixture + .executor + .run(move || config_store.load_workspace_config(WORKSPACE)), + ) + .await + .unwrap() + .unwrap(); + assert_ne!( + state.snapshot.digest, current.snapshot.digest, + "publication must observe the committed tree" + ); + assert!( + state + .snapshot + .entries + .keys() + .any(|path| path.to_string() == "notes/owned-publication.txt") + ); + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert_eq!( + observed(tokio::spawn(async { "control" })).await.unwrap(), + "control" + ); + assert!(!reply.upstream_released.load(Ordering::SeqCst)); + drop(commit); + assert_eq!(leases.load(Ordering::SeqCst), 1); + reply.finish(); + observed(fixture.host_executor.drain()).await.unwrap(); + observed(fixture.executor.drain()).await.unwrap(); + assert_eq!(leases.load(Ordering::SeqCst), 0); + assert_eq!( + fixture + .api + .config_store + .load_workspace_config(WORKSPACE) + .unwrap() + .unwrap() + .snapshot + .digest, + state.snapshot.digest, + "failed best-effort publication must not repeat or roll back the commit" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn repository_provider_reads_do_not_consume_the_sql_execution_queue() { + let fixture = fixture().await; + let (entered, waiting) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + fixture + .executor + .dispatch(move || { + let _ = entered.send(()); + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + observed(waiting).await.unwrap(); + fixture.executor.dispatch(|| {}).unwrap(); + let probe = fixture.executor.run(|| Ok::<_, Error>(())).await; + assert!(matches!(probe, Err(Error::DbCapacityExceeded))); + // Empty catalog / absent key are deliberate boundary inputs: provider + // reads keep their ordinary response even when SQL job admission is full. + let list = observed(list_repositories(State(fixture.api.clone()))) + .await + .unwrap(); + assert!(list.0.items.is_empty()); + let detail = observed(repository_detail( + State(fixture.api.clone()), + AxumPath("missing".into()), + )) + .await + .unwrap_err(); + response_json(detail.into_response(), StatusCode::NOT_FOUND).await; + let log = observed(repository_log( + State(fixture.api.clone()), + AxumPath("missing".into()), + Query(LogQuery { limit: Some(1) }), + )) + .await + .unwrap_err(); + response_json(log.into_response(), StatusCode::NOT_FOUND).await; + let _ = release.send(()); + observed(fixture.executor.drain()).await.unwrap(); + observed(fixture.host_executor.drain()).await.unwrap(); +} diff --git a/crates/workspace-server/src/server_ticket_case_signed_tests.rs b/crates/workspace-server/src/server_ticket_case_signed_tests.rs index 78ba317b2..8f7c38b52 100644 --- a/crates/workspace-server/src/server_ticket_case_signed_tests.rs +++ b/crates/workspace-server/src/server_ticket_case_signed_tests.rs @@ -724,7 +724,7 @@ async fn catalog_case_dispatch_authenticates_and_routes_the_canonical_workspace_ reason: ticket::TicketClosureReason::Completed, }, ); - let catalog = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); + let catalog = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); // Retain only this in-process fixture: do not create a real Runtime or a // background Orchestrator hook while testing catalog authentication/routing. catalog diff --git a/crates/workspace-server/src/server_ticket_tags_tests.rs b/crates/workspace-server/src/server_ticket_tags_tests.rs index b636a8356..1608982bd 100644 --- a/crates/workspace-server/src/server_ticket_tags_tests.rs +++ b/crates/workspace-server/src/server_ticket_tags_tests.rs @@ -529,7 +529,7 @@ async fn runtime_tag_mutations_reject_invalid_proofs_and_changed_binding_authori *tampered.body_mut() = Body::from(format!("{} ", input)); runtime_tag_response(app.clone(), tampered, StatusCode::UNAUTHORIZED).await; } - let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); let original = api .store .get_workspace_runtime_binding(TEST_WORKSPACE_ID, "runtime-tags") diff --git a/crates/workspace-server/src/server_workspace_config.rs b/crates/workspace-server/src/server_workspace_config.rs index 0e0a54959..dfaa104af 100644 --- a/crates/workspace-server/src/server_workspace_config.rs +++ b/crates/workspace-server/src/server_workspace_config.rs @@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into) -> Failure { fn mutation_error(error: Error) -> Failure { // Storage/service failures at a mutation boundary may include a commit // acknowledgement failure. Never describe those as a guaranteed rollback. - if matches!(&error, Error::Sqlite(_) | Error::Store(_)) { + if matches!( + &error, + Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown + ) { unknown() } else { pre_error(error) @@ -59,7 +62,7 @@ fn unknown() -> Failure { ..fail(500, "outcome_unknown") } } -pub(super) fn identity( +pub(super) async fn identity( api: &WorkspaceApi, context: &server_api::ServerRequestContext, workspace_id: &str, @@ -67,7 +70,9 @@ pub(super) fn identity( validate_workspace_scope(api, workspace_id).map_err(pre_error)?; let headers = current_worker_contract_headers(context).map_err(|_| fail(403, "access_denied"))?; - current_worker_identity(api, workspace_id, &headers).map_err(pre_error) + authenticate_worker_mutation_source_owned(api, workspace_id, &headers) + .await + .map_err(pre_error) } async fn lock( api: &WorkspaceApi, @@ -186,9 +191,23 @@ pub(super) async fn get( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> ConfigResult> { - let _guard = lock(api, worker).await?; - let grant = grant_for_worker(api, worker)?; - Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true))) + let guard = lock(api, worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let worker = worker.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + let grant = grant_for_worker(&api, &worker)?; + Ok::<_, Failure>( + active_link(&api, &worker, &grant, None)? + .map(|link| attachment(&grant, &link, true)), + ) + })()) + }) + .await + .map_err(pre_error)? } pub(super) async fn attach( api: &WorkspaceApi, @@ -198,46 +217,59 @@ pub(super) async fn attach( if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) { return Err(fail(400, "invalid_alias")); } - let _guard = lock(api, worker).await?; - let grant = grant_for_worker(api, worker)?; - let access = request.access.unwrap_or(grant.access); - if access == Access::ReadWrite && grant.access != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - if let Some(mut link) = active_link(api, worker, &grant, None)? { - // Same active alias/identity is idempotent; explicit write never silently downgrades. - if request.access == Some(Access::ReadWrite) - && effective_access(&grant, &link) != Access::ReadWrite - { - return Err(fail(403, "access_denied")); - } - if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly { - link.capabilities = access_capabilities(access); - link = api - .store - .attach_worker_workdir(&link) - .map_err(mutation_error)?; - } - return Ok(attachment(&grant, &link, true)); - } - let link = api - .store - .attach_worker_workdir(&WorkerWorkdirLinkRecord { - connection_id: String::new(), - workspace_id: grant.workspace_id.clone(), - worker: worker.clone(), - workdir_id: grant.working_directory_id.clone(), - alias: ALIAS.into(), - capabilities: access_capabilities(access), - linked_at: now_registry_timestamp(), - unlinked_at: None, + let guard = lock(api, worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let worker = worker.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + let grant = grant_for_worker(&api, &worker)?; + let access = request.access.unwrap_or(grant.access); + if access == Access::ReadWrite && grant.access != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + if let Some(mut link) = active_link(&api, &worker, &grant, None)? { + // Same active alias/identity is idempotent; explicit write never silently downgrades. + if request.access == Some(Access::ReadWrite) + && effective_access(&grant, &link) != Access::ReadWrite + { + return Err(fail(403, "access_denied")); + } + if access == Access::ReadOnly + && effective_access(&grant, &link) != Access::ReadOnly + { + link.capabilities = access_capabilities(access); + link = api + .store + .attach_worker_workdir(&link) + .map_err(mutation_error)?; + } + return Ok(attachment(&grant, &link, true)); + } + let link = api + .store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: grant.workspace_id.clone(), + worker: worker.clone(), + workdir_id: grant.working_directory_id.clone(), + alias: ALIAS.into(), + capabilities: access_capabilities(access), + linked_at: now_registry_timestamp(), + unlinked_at: None, + }) + .map_err(mutation_error)?; + // No Runtime filesystem/session synchronization for a logical config target. + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(attachment(&grant, &link, false)) + })()) }) - .map_err(mutation_error)?; - // No Runtime filesystem/session synchronization for a logical config target. - api.worker_projection - .refresh(worker) - .map_err(|_| unknown())?; - Ok(attachment(&grant, &link, false)) + .await + .map_err(mutation_error)? } fn resolve( api: &WorkspaceApi, @@ -313,9 +345,22 @@ pub(super) async fn observe( for path in &request.paths { path_valid(path, true)?; } - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, _guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -427,9 +472,22 @@ pub(super) async fn read( request: WorkspaceConfigReadRequest, ) -> ConfigResult { path_valid(&request.path, false)?; - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, _guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -482,12 +540,25 @@ pub(super) async fn commit( server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?, } } - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - if effective_access(&grant, &link) != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + if effective_access(&grant, &link) != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -501,17 +572,19 @@ pub(super) async fn commit( // Evaluation/semantic validation are pure preparation. Bound this stage // without ever dispatching persistence in a timed-out background task. let preparation_api = api.clone(); - let preparation = tokio::task::spawn_blocking(move || { - prepare_workspace_config_tree( + let executor = api.store.db_executor(); + let preparation = executor.run(move || { + // Timeout/cancellation releases exclusion only after actual preparation ends. + let candidate = prepare_workspace_config_tree( &preparation_api, &preparation_api.config.workspace_id, &canonical, - ) + )?; + Ok::<_, Error>((candidate, guard)) }); - let candidate = tokio::time::timeout(DEADLINE, preparation) + let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation) .await .map_err(|_| fail(503, "evaluation_timeout"))? - .map_err(|_| fail(500, "evaluation_unavailable"))? .map_err(pre_error)?; if candidate.base_digest != state.snapshot.digest || candidate.base_toolchain_fingerprint != state.contract.fingerprint @@ -520,15 +593,78 @@ pub(super) async fn commit( { return Err(fail(409, "stale_validator")); } - // Recheck durable authority before the first auxiliary/canonical effects. - let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?; - if current_grant != grant || current_link.capabilities != link.capabilities { - return Err(fail(409, "authority_changed")); + let persistence_api = api.clone(); + let worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let expected_grant = grant.clone(); + let expected_link = link.clone(); + let (state, publication) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (current_grant, current_link) = + resolve(&persistence_api, &worker, &connection_id)?; + if current_grant != expected_grant + || current_link.capabilities != expected_link.capabilities + { + return Err(fail(409, "authority_changed")); + } + // Preserve the shared helper's ordering and atomic CAS, but never + // run its remote Runtime publication on the sole DB worker. + project_repository_access_candidate( + &*persistence_api.store, + &persistence_api.repository_secrets, + &persistence_api.config.workspace_id, + &candidate, + ) + .map_err(|_| unknown())?; + let state = persistence_api + .config_store + .commit_evaluated_workspace_config( + &persistence_api.config.workspace_id, + &candidate, + ) + .map_err(|_| unknown())?; + let publication = persistence_api + .prompt_projection_cache + .resolve(&persistence_api.config.workspace_id, &state) + .ok() + .and_then(|projection| { + // Dispatch from the actual persistence owner even if its + // waiter was cancelled. Runtime I/O owns exclusion, not + // a DB worker or an open SQL transaction. + let (complete, publication) = tokio::sync::oneshot::channel(); + let runtime = persistence_api.runtime.clone(); + let admission = persistence_api.host_executors.host().and_then(|executor| { + executor.dispatch(move || { + let _guard = guard; + let _diagnostics = runtime + .observe_workspace_prompt_projection((*projection).clone()); + let _ = complete.send(()); + }) + }); + match admission { + Ok(()) => Some(publication), + Err(error) => { + tracing::warn!( + workspace_id = %persistence_api.config.workspace_id, + %error, + "Workspace config committed but Runtime publication was not admitted" + ); + None + } + } + }); + Ok((state, publication)) + })()) + }) + .await + .map_err(mutation_error)??; + if let Some(publication) = publication { + // Publication is best effort and cannot turn an acknowledged CAS into an error. + let _ = publication.await; } - // From this boundary repository credential projection may have side effects. - // Never falsely claim rollback or automatically replay a failed/uncertain save. - let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate) - .map_err(|_| unknown())?; Ok(WorkspaceConfigCommitResponse { validator: validator(&grant, &link, &state, &schema_fingerprint, ""), digest: state.snapshot.digest, @@ -544,37 +680,61 @@ pub(super) async fn create_grant( .await .map_err(pre_error)?; let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id); - let _guard = lock(api, &worker).await?; - let summary = api - .runtime - .worker(&worker) - .map_err(|_| fail(404, "worker_not_found"))?; + let guard = lock(api, &worker).await?; + let runtime = api.runtime.clone(); + let lookup_worker = worker.clone(); + let (summary, guard) = api + .host_executors + .host() + .map_err(pre_error)? + .run(move || { + Ok::<_, Error>((|| { + let summary = runtime + .worker(&lookup_worker) + .map_err(|_| fail(404, "worker_not_found"))?; + Ok::<_, Failure>((summary, guard)) + })()) + }) + .await + .map_err(pre_error)??; if summary.workspace.workspace_id.as_deref() != Some(workspace_id) { return Err(fail(403, "access_denied")); } - if let Some(grant) = api - .store - .current_workspace_config_grant(workspace_id, &worker) - .map_err(pre_error)? - { - if grant.access != request.access { - return Err(fail(409, "grant_exists_revoke_first")); - } - return Ok(grant); - } - let grant = WorkspaceConfigGrantResponse { - grant_id: Uuid::now_v7().to_string(), - workspace_id: workspace_id.into(), - runtime_id: worker.runtime_id, - worker_id: worker.worker_id, - working_directory_id: Uuid::now_v7().to_string(), - access: request.access, - revoked: false, - }; - api.store - .create_workspace_config_grant(&grant, &actor.account_id) - .map_err(mutation_error)?; - Ok(grant) + let executor = api.store.db_executor(); + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let account_id = actor.account_id.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + if let Some(grant) = api + .store + .current_workspace_config_grant(&workspace_id, &worker) + .map_err(pre_error)? + { + if grant.access != request.access { + return Err(fail(409, "grant_exists_revoke_first")); + } + return Ok(grant); + } + let grant = WorkspaceConfigGrantResponse { + grant_id: Uuid::now_v7().to_string(), + workspace_id, + runtime_id: worker.runtime_id, + worker_id: worker.worker_id, + working_directory_id: Uuid::now_v7().to_string(), + access: request.access, + revoked: false, + }; + api.store + .create_workspace_config_grant(&grant, &account_id) + .map_err(mutation_error)?; + Ok(grant) + })()) + }) + .await + .map_err(mutation_error)? } pub(super) async fn revoke_grant( api: &WorkspaceApi, @@ -585,39 +745,60 @@ pub(super) async fn revoke_grant( require_workspace_owner(api, workspace_id, actor, "Workspace config grants") .await .map_err(pre_error)?; + let lookup_api = api.clone(); + let lookup_workspace_id = workspace_id.to_owned(); + let lookup_grant_id = grant_id.to_owned(); let mut grant = api .store - .get_workspace_config_grant(workspace_id, grant_id) + .db_executor() + .run(move || { + lookup_api + .store + .get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id) + }) + .await .map_err(pre_error)? .ok_or_else(|| fail(404, "not_found"))?; let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id); - let _guard = lock(api, &worker).await?; - api.store - .revoke_workspace_config_grant(workspace_id, grant_id) - .map_err(mutation_error)?; - grant.revoked = true; - // Revocation is durable before unlink; a failed cleanup cannot restore access. - if let Some(link) = api - .store - .list_workdir_worker_links(workspace_id, &grant.working_directory_id) - .map_err(|_| unknown())? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.worker == worker) - { - api.store - .detach_worker_workdir_connection( - workspace_id, - &worker, - &link.alias, - &link.connection_id, - &now_registry_timestamp(), - ) - .map_err(|_| unknown())?; - } - api.worker_projection - .refresh(&worker) - .map_err(|_| unknown())?; - Ok(grant) + let guard = lock(api, &worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let grant_id = grant_id.to_owned(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + api.store + .revoke_workspace_config_grant(&workspace_id, &grant_id) + .map_err(mutation_error)?; + grant.revoked = true; + // Revocation is durable before unlink; a failed cleanup cannot restore access. + if let Some(link) = api + .store + .list_workdir_worker_links(&workspace_id, &grant.working_directory_id) + .map_err(|_| unknown())? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.worker == worker) + { + api.store + .detach_worker_workdir_connection( + &workspace_id, + &worker, + &link.alias, + &link.connection_id, + &now_registry_timestamp(), + ) + .map_err(|_| unknown())?; + } + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(grant) + })()) + }) + .await + .map_err(mutation_error)? } pub(super) fn summary( @@ -655,7 +836,11 @@ pub(super) fn summary( mod tests { use super::*; #[test] - fn workspace_config_mutation_storage_failures_are_conservatively_unknown() { + fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() { + assert_eq!( + mutation_error(Error::DbOperationOutcomeUnknown).classification, + Classification::Unknown + ); assert_eq!( mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification, Classification::Unknown diff --git a/crates/workspace-server/src/server_workspace_config_tests.rs b/crates/workspace-server/src/server_workspace_config_tests.rs index ac11e7766..1c932be1a 100644 --- a/crates/workspace-server/src/server_workspace_config_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_tests.rs @@ -318,7 +318,7 @@ mod workspace_config_integration { }; for scoped in [false, true] { let app = if scoped { - workspace_server_router(WorkspaceServerApi::new( + workspace_server_router(WorkspaceServerApi::new_for_test( api.config.clone(), api.store.clone(), )) @@ -803,9 +803,9 @@ mod workspace_config_integration { .supports(workdir::WorkdirSessionCapability::Command) ); let lock = current_worker_session_lock(api, &fixture.worker); - let _guard = lock.lock().await; + let _guard = Arc::new(lock.lock_owned().await); assert!( - open_current_worker_workdir_session_locked(api, &fixture.worker, &link) + open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard) .await .is_err() ); @@ -1385,7 +1385,7 @@ mod workspace_config_integration { let api = &fixture.api; let granted = grant(api, &fixture.worker, Access::ReadWrite).await; let attached = attach(api, &fixture.worker).await; - let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); assert_eq!( restarted .get_workspace_config_grant(TEST_WORKSPACE_ID, &granted.grant_id) diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index b5a229f55..1e989edd8 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -800,7 +800,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w let base = format!("http://{addr}"); let mut config = test_server_config(root.path()); config.backend_base_url = Some("server-test".into()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); let captured = Arc::new(Mutex::new(None)); let abort = Arc::new(Mutex::new(None)); let factory = ConfigCaptureFactory { diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 3e44c9081..4585493d5 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -1131,6 +1131,10 @@ impl WorkspaceResourceKind { #[async_trait] pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { + fn db_executor(&self) -> crate::db_executor::DbExecutor { + crate::db_executor::DbExecutor::shared() + } + async fn schema_version(&self) -> Result; fn resource_key( &self, @@ -2257,6 +2261,7 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { #[derive(Clone)] pub struct SqliteWorkspaceStore { conn: Arc>, + db_executor: crate::db_executor::DbExecutor, } impl SqliteWorkspaceStore { @@ -2297,12 +2302,50 @@ impl SqliteWorkspaceStore { } pub fn from_connection(conn: Connection) -> Result { + Self::from_connection_with_executor(conn, crate::db_executor::DbExecutor::shared()) + } + + fn from_connection_with_executor( + conn: Connection, + db_executor: crate::db_executor::DbExecutor, + ) -> Result { prepare_connection(&conn)?; Ok(Self { conn: Arc::new(Mutex::new(conn)), + db_executor, }) } + // Ordinary fixtures own the same finite budget as production, not another + // fixture's admission/SQL/drain lifetime or a permanent global worker. + #[cfg(test)] + pub(crate) fn open_for_test(path: impl AsRef) -> Result { + Self::from_connection_for_test(Connection::open(path)?) + } + + #[cfg(test)] + pub(crate) fn in_memory_for_test() -> Result { + Self::from_connection_for_test(Connection::open_in_memory()?) + } + + #[cfg(test)] + pub(crate) fn from_connection_for_test(conn: Connection) -> Result { + Self::from_connection_with_executor( + conn, + crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits::default())?, + ) + } + + /// Override execution limits for a hermetic fixture; subsequent clones share it. + pub fn with_db_executor(mut self, executor: crate::db_executor::DbExecutor) -> Self { + self.db_executor = executor; + self + } + + pub fn db_executor(&self) -> crate::db_executor::DbExecutor { + self.db_executor.clone() + } + pub(crate) fn with_conn(&self, f: impl FnOnce(&Connection) -> Result) -> Result { let conn = self .conn @@ -2322,12 +2365,16 @@ impl SqliteWorkspaceStore { f(&mut conn) } - pub(crate) fn with_transaction( + pub(crate) fn with_immediate_transaction( &self, f: impl FnOnce(&rusqlite::Transaction<'_>) -> Result, ) -> Result { self.with_conn_mut(|conn| { - let tx = conn.transaction()?; + // These callers read authority/journal state before mutating it. + // Acquire write ownership before the first read: a DEFERRED read + // upgrade against another connection's writer can return BUSY + // without invoking the busy handler. Never retry the domain mutation. + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let value = f(&tx)?; tx.commit()?; Ok(value) @@ -2388,7 +2435,7 @@ impl SqliteWorkspaceStore { observed_token: Option<&str>, explicit_request_id: Option<&str>, ) -> Result { - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { let existing: Option<(String, String)> = if let Some(id) = explicit_request_id { tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4", params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()? @@ -3872,8 +3919,16 @@ fn insert_workspace_runtime_binding_audit( #[async_trait] impl ControlPlaneStore for SqliteWorkspaceStore { + fn db_executor(&self) -> crate::db_executor::DbExecutor { + self.db_executor.clone() + } + async fn schema_version(&self) -> Result { - self.with_conn(current_schema_version) + let store = self.clone(); + + self.db_executor + .run(move || store.with_conn(current_schema_version)) + .await } fn resource_key( @@ -4005,100 +4060,114 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()> { - self.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - #[cfg(test)] - if record.owner_account_id == "owner-account" { - // Unrelated store tests use this explicit fixture identity. Keep the - // production owner contract strict while giving those fixtures a real - // User Account row instead of reviving ownerless Workspace setup. - tx.execute( - "INSERT OR IGNORE INTO accounts ( - account_id, kind, handle, display_name, created_at, updated_at - ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", - params![record.owner_account_id.as_str(), record.created_at.as_str()], - )?; - } - let owner_kind = tx - .query_row( - "SELECT kind FROM accounts WHERE account_id = ?1", - params![record.owner_account_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if owner_kind.as_deref() != Some("user") { - return Err(Error::Store(format!( - "Workspace owner `{}` must reference an existing User Account", - record.owner_account_id - ))); - } - let current_owner = tx - .query_row( - "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", - params![record.workspace_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if current_owner - .as_deref() - .is_some_and(|owner| owner != record.owner_account_id) - { - return Err(Error::Store(format!( - "Workspace `{}` owner is immutable through upsert", - record.workspace_id - ))); - } - tx.execute( - r#"INSERT INTO workspaces ( - workspace_id, owner_account_id, display_name, state, created_at, updated_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(workspace_id) DO UPDATE SET - display_name = excluded.display_name, - state = excluded.state, - updated_at = excluded.updated_at"#, - params![ - record.workspace_id, - record.owner_account_id, - record.display_name, - record.state, - record.created_at, - record.updated_at, - ], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_memory_settings ( - workspace_id, language, created_at, updated_at - ) VALUES (?1, 'English', ?2, ?3)"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_signing_identities ( - workspace_id, key_id, algorithm, public_key, public_key_fingerprint, - private_material_ref, state, created_at, provisioned_at, updated_at - ) VALUES ( - ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, - 'workspace-signing/' || ?1 || '/ed25519-v1', - 'pending_provisioning', ?2, NULL, ?3 - )"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.commit()?; - Ok(()) - })?; - self.materialize_workspace_config(&record.workspace_id, &record.created_at) + let store = self.clone(); + let record = record.clone(); + self.db_executor + .run(move || { + let store = &store; + let record = &record; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + #[cfg(test)] + if record.owner_account_id == "owner-account" { + // Unrelated store tests use this explicit fixture identity. Keep the + // production owner contract strict while giving those fixtures a real + // User Account row instead of reviving ownerless Workspace setup. + tx.execute( + "INSERT OR IGNORE INTO accounts ( + account_id, kind, handle, display_name, created_at, updated_at + ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", + params![record.owner_account_id.as_str(), record.created_at.as_str()], + )?; + } + let owner_kind = tx + .query_row( + "SELECT kind FROM accounts WHERE account_id = ?1", + params![record.owner_account_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if owner_kind.as_deref() != Some("user") { + return Err(Error::Store(format!( + "Workspace owner `{}` must reference an existing User Account", + record.owner_account_id + ))); + } + let current_owner = tx + .query_row( + "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", + params![record.workspace_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if current_owner + .as_deref() + .is_some_and(|owner| owner != record.owner_account_id) + { + return Err(Error::Store(format!( + "Workspace `{}` owner is immutable through upsert", + record.workspace_id + ))); + } + tx.execute( + r#"INSERT INTO workspaces ( + workspace_id, owner_account_id, display_name, state, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(workspace_id) DO UPDATE SET + display_name = excluded.display_name, + state = excluded.state, + updated_at = excluded.updated_at"#, + params![ + record.workspace_id, + record.owner_account_id, + record.display_name, + record.state, + record.created_at, + record.updated_at, + ], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_memory_settings ( + workspace_id, language, created_at, updated_at + ) VALUES (?1, 'English', ?2, ?3)"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_signing_identities ( + workspace_id, key_id, algorithm, public_key, public_key_fingerprint, + private_material_ref, state, created_at, provisioned_at, updated_at + ) VALUES ( + ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, + 'workspace-signing/' || ?1 || '/ed25519-v1', + 'pending_provisioning', ?2, NULL, ?3 + )"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.commit()?; + Ok(()) + })?; + store.materialize_workspace_config(&record.workspace_id, &record.created_at) + }) + .await } async fn get_workspace(&self, workspace_id: &str) -> Result> { - self.with_conn(|conn| { - conn.query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional() - .map_err(Error::from) - }) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor.run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + store.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional() + .map_err(Error::from) + }) + }).await } async fn update_workspace_display_name( @@ -4107,58 +4176,68 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_updated_at: &str, display_name: &str, ) -> Result> { - validate_identifier("workspace_id", workspace_id)?; - validate_non_empty("expected_updated_at", expected_updated_at)?; - validate_non_empty("display_name", display_name)?; - self.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let current = tx - .query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional()?; - let Some(current) = current else { - tx.commit()?; - return Ok(None); - }; - if current.updated_at != expected_updated_at { - tx.commit()?; - return Ok(None); - } - if current.display_name == display_name { - tx.commit()?; - return Ok(Some(current)); - } + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let expected_updated_at = expected_updated_at.to_owned(); + let display_name = display_name.to_owned(); + self.db_executor.run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let expected_updated_at = expected_updated_at.as_str(); + let display_name = display_name.as_str(); + validate_identifier("workspace_id", workspace_id)?; + validate_non_empty("expected_updated_at", expected_updated_at)?; + validate_non_empty("display_name", display_name)?; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current = tx + .query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional()?; + let Some(current) = current else { + tx.commit()?; + return Ok(None); + }; + if current.updated_at != expected_updated_at { + tx.commit()?; + return Ok(None); + } + if current.display_name == display_name { + tx.commit()?; + return Ok(Some(current)); + } - let now = chrono::Utc::now(); - let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) - .ok() - .map(|previous| previous.with_timezone(&chrono::Utc)) - .filter(|previous| *previous >= now) - .map(|previous| previous + chrono::Duration::nanoseconds(1)) - .unwrap_or(now) - .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); - let changed = tx.execute( - r#"UPDATE workspaces - SET display_name = ?3, updated_at = ?4 - WHERE workspace_id = ?1 AND updated_at = ?2"#, - params![workspace_id, expected_updated_at, display_name, updated_at], - )?; - if changed != 1 { + let now = chrono::Utc::now(); + let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) + .ok() + .map(|previous| previous.with_timezone(&chrono::Utc)) + .filter(|previous| *previous >= now) + .map(|previous| previous + chrono::Duration::nanoseconds(1)) + .unwrap_or(now) + .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + let changed = tx.execute( + r#"UPDATE workspaces + SET display_name = ?3, updated_at = ?4 + WHERE workspace_id = ?1 AND updated_at = ?2"#, + params![workspace_id, expected_updated_at, display_name, updated_at], + )?; + if changed != 1 { + tx.commit()?; + return Ok(None); + } + let updated = WorkspaceRecord { + display_name: display_name.to_string(), + updated_at, + ..current + }; tx.commit()?; - return Ok(None); - } - let updated = WorkspaceRecord { - display_name: display_name.to_string(), - updated_at, - ..current - }; - tx.commit()?; - Ok(Some(updated)) - }) + Ok(Some(updated)) + }) + }).await } fn create_workspace_bootstrap( @@ -4678,7 +4757,17 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result> { - SqliteWorkspaceStore::get_workspace_runtime_binding(self, workspace_id, runtime_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::get_workspace_runtime_binding(store, workspace_id, runtime_id) + }) + .await } async fn list_workspace_runtime_bindings( @@ -4686,7 +4775,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, include_revoked: bool, ) -> Result> { - SqliteWorkspaceStore::list_workspace_runtime_bindings(self, workspace_id, include_revoked) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + SqliteWorkspaceStore::list_workspace_runtime_bindings( + store, + workspace_id, + include_revoked, + ) + }) + .await } async fn has_other_active_workspace_runtime_binding( @@ -4694,11 +4795,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( - self, - workspace_id, - runtime_id, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + ) + }) + .await } async fn delete_workspace_runtime_binding( @@ -4706,7 +4817,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - SqliteWorkspaceStore::delete_workspace_runtime_binding(self, workspace_id, runtime_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::delete_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + ) + }) + .await } async fn reserve_runtime_removal( @@ -4718,22 +4843,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: &str, config_digest: &str, ) -> Result { - SqliteWorkspaceStore::reserve_runtime_removal( - self, - workspace_id, - runtime_id, - operation_id, - request_fingerprint, - expected_binding_id, - config_digest, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let operation_id = operation_id.to_owned(); + let request_fingerprint = request_fingerprint.to_owned(); + let expected_binding_id = expected_binding_id.to_owned(); + let config_digest = config_digest.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let operation_id = operation_id.as_str(); + let request_fingerprint = request_fingerprint.as_str(); + let expected_binding_id = expected_binding_id.as_str(); + let config_digest = config_digest.as_str(); + SqliteWorkspaceStore::reserve_runtime_removal( + store, + workspace_id, + runtime_id, + operation_id, + request_fingerprint, + expected_binding_id, + config_digest, + ) + }) + .await } async fn get_runtime_removal( &self, operation_id: &str, ) -> Result> { - SqliteWorkspaceStore::get_runtime_removal(self, operation_id) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::get_runtime_removal(store, operation_id) + }) + .await } async fn mark_runtime_removal_failed( @@ -4741,14 +4892,36 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, failure_category: &str, ) -> Result { - SqliteWorkspaceStore::mark_runtime_removal_failed(self, operation_id, failure_category) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + let failure_category = failure_category.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + let failure_category = failure_category.as_str(); + SqliteWorkspaceStore::mark_runtime_removal_failed( + store, + operation_id, + failure_category, + ) + }) + .await } async fn commit_runtime_binding_removal( &self, operation_id: &str, ) -> Result { - SqliteWorkspaceStore::commit_runtime_binding_removal(self, operation_id) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::commit_runtime_binding_removal(store, operation_id) + }) + .await } async fn complete_runtime_removal( @@ -4756,18 +4929,34 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, runtime_registration_removed: bool, ) -> Result { - SqliteWorkspaceStore::complete_runtime_removal( - self, - operation_id, - runtime_registration_removed, - ) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::complete_runtime_removal( + store, + operation_id, + runtime_registration_removed, + ) + }) + .await } async fn list_resumable_runtime_removals( &self, workspace_id: &str, ) -> Result> { - SqliteWorkspaceStore::list_resumable_runtime_removals(self, workspace_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + SqliteWorkspaceStore::list_resumable_runtime_removals(store, workspace_id) + }) + .await } async fn upsert_workspace_runtime_binding_record( @@ -4775,7 +4964,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record: WorkspaceRuntimeBinding, replace: bool, ) -> Result { - SqliteWorkspaceStore::upsert_workspace_runtime_binding(self, record, replace) + let store = self.clone(); + + self.db_executor + .run(move || { + let store = &store; + SqliteWorkspaceStore::upsert_workspace_runtime_binding(store, record, replace) + }) + .await } async fn revoke_workspace_runtime_binding_record( @@ -4784,12 +4980,24 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, revoked_at: &str, ) -> Result { - SqliteWorkspaceStore::revoke_workspace_runtime_binding( - self, - workspace_id, - runtime_id, - revoked_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let revoked_at = revoked_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let revoked_at = revoked_at.as_str(); + SqliteWorkspaceStore::revoke_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + revoked_at, + ) + }) + .await } async fn put_workspace_runtime_binding_key( @@ -4798,12 +5006,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: Option<&str>, actor_account_id: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - SqliteWorkspaceStore::put_workspace_runtime_binding_key( - self, - record, - expected_binding_id, - actor_account_id, - ) + let store = self.clone(); + let expected_binding_id = expected_binding_id.map(str::to_owned); + let actor_account_id = actor_account_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let expected_binding_id = expected_binding_id.as_deref(); + let actor_account_id = actor_account_id.as_str(); + SqliteWorkspaceStore::put_workspace_runtime_binding_key( + store, + record, + expected_binding_id, + actor_account_id, + ) + }) + .await } async fn update_workspace_runtime_binding_metadata( @@ -4814,14 +5032,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore { base_url: &str, updated_at: &str, ) -> Result { - SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( - self, - workspace_id, - runtime_id, - display_name, - base_url, - updated_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let display_name = display_name.to_owned(); + let base_url = base_url.to_owned(); + let updated_at = updated_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let display_name = display_name.as_str(); + let base_url = base_url.as_str(); + let updated_at = updated_at.as_str(); + SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( + store, + workspace_id, + runtime_id, + display_name, + base_url, + updated_at, + ) + }) + .await } async fn revoke_workspace_runtime_binding_key( @@ -4832,14 +5066,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore { actor_account_id: &str, revoked_at: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( - self, - workspace_id, - runtime_id, - expected_binding_id, - actor_account_id, - revoked_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let expected_binding_id = expected_binding_id.to_owned(); + let actor_account_id = actor_account_id.to_owned(); + let revoked_at = revoked_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let expected_binding_id = expected_binding_id.as_str(); + let actor_account_id = actor_account_id.as_str(); + let revoked_at = revoked_at.as_str(); + SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( + store, + workspace_id, + runtime_id, + expected_binding_id, + actor_account_id, + revoked_at, + ) + }) + .await } async fn list_workspace_runtime_binding_audit( @@ -4848,12 +5098,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, limit: usize, ) -> Result> { - SqliteWorkspaceStore::list_workspace_runtime_binding_audit( - self, - workspace_id, - runtime_id, - limit, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::list_workspace_runtime_binding_audit( + store, + workspace_id, + runtime_id, + limit, + ) + }) + .await } async fn consume_worker_mutation_source_jti( @@ -4865,21 +5125,35 @@ impl ControlPlaneStore for SqliteWorkspaceStore { now_seconds: u64, consumed_at: &str, ) -> Result { - self.with_conn(|conn| { - let transaction = conn.unchecked_transaction()?; - transaction.execute( - "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", - params![now_seconds], - )?; - let inserted = transaction.execute( - r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( - workspace_id, runtime_id, jti, expires_at, consumed_at - ) VALUES (?1, ?2, ?3, ?4, ?5)"#, - params![workspace_id, runtime_id, jti, expires_at, consumed_at], - )?; - transaction.commit()?; - Ok(inserted == 1) - }) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let jti = jti.to_owned(); + let consumed_at = consumed_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let jti = jti.as_str(); + let consumed_at = consumed_at.as_str(); + store.with_conn(|conn| { + let transaction = conn.unchecked_transaction()?; + transaction.execute( + "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", + params![now_seconds], + )?; + let inserted = transaction.execute( + r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( + workspace_id, runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, ?2, ?3, ?4, ?5)"#, + params![workspace_id, runtime_id, jti, expires_at, consumed_at], + )?; + transaction.commit()?; + Ok(inserted == 1) + }) + }) + .await } fn plan_worker_removal( @@ -14686,7 +14960,7 @@ mod tests { fn drive_grants_migrate_and_preserve_audit_through_revoke_worker_delete_and_restart() { let t = tempfile::tempdir().unwrap(); let path = t.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let legacy = Connection::open_in_memory().unwrap(); configure_sqlite(&legacy).unwrap(); legacy.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY); @@ -14776,7 +15050,7 @@ mod tests { Some(other) ); drop(store); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( store .get_workspace_drive_grant("drive-ws", &first.grant_id) @@ -14811,7 +15085,7 @@ mod tests { } #[test] fn drive_grants_baseline_constraints_and_worker_lifetime_fail_closed() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let proposed = drive_grant_fixture(&store); let first = store.create_workspace_drive_grant(&proposed).unwrap(); store.with_conn(|c| { @@ -14847,7 +15121,7 @@ mod tests { fn schema_v74_migrates_historical_worker_projection_rows_without_counter_state() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("workspace.db"); - drop(SqliteWorkspaceStore::open(&path).unwrap()); + drop(SqliteWorkspaceStore::open_for_test(&path).unwrap()); let conn = Connection::open(&path).unwrap(); configure_sqlite(&conn).unwrap(); conn.execute_batch( @@ -15172,7 +15446,7 @@ mod tests { #[tokio::test] async fn external_workdir_grant_persists_typed_source_and_fences_generations() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .upsert_account(&AccountRecord { account_id: "owner".to_string(), @@ -15544,7 +15818,7 @@ mod tests { fn runtime_removal_test_store() -> (tempfile::TempDir, SqliteWorkspaceStore) { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -15928,7 +16202,7 @@ mod tests { let path = temp.path().join("server.db"); prepare_retained_schema(&path, 59); - let migrated = SqliteWorkspaceStore::open(&path).unwrap(); + let migrated = SqliteWorkspaceStore::open_for_test(&path).unwrap(); migrated .with_conn(|conn| { let table_count: i64 = conn.query_row( @@ -15964,7 +16238,7 @@ mod tests { let path = temp.path().join("server.db"); prepare_retained_schema(&path, 60); - drop(SqliteWorkspaceStore::open(&path).unwrap()); + drop(SqliteWorkspaceStore::open_for_test(&path).unwrap()); let conn = Connection::open(&path).unwrap(); assert_eq!( current_schema_version(&conn).unwrap(), @@ -15994,7 +16268,7 @@ mod tests { #[test] fn schema_v59_runtime_removal_migration_rolls_back_partial_ddl_and_marker() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap(); store .with_conn(|conn| { let trigger_names = { @@ -16258,7 +16532,7 @@ mod tests { fn fresh_database_uses_only_the_canonical_workspace_baseline() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { let migrations = conn @@ -16406,7 +16680,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .collect::>(), (83..=LATEST_SCHEMA_VERSION).collect::>() ); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let assignment = store .get_active_ticket_worker_assignment("workspace-a", "ticket") .unwrap() @@ -16877,7 +17151,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let applied = SqliteWorkspaceStore::migrate_database(&path).unwrap(); assert_eq!(applied, plan); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -17426,7 +17700,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "migrated databases must reject incomplete singleton leases" ); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let legacy = RuntimeWorkerRef::new("legacy-runtime", "legacy-orchestrator"); let lease = store .require_current_worker_singleton_owner("workspace-a", &legacy) @@ -17614,7 +17888,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -17715,7 +17989,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(conn); - let error = SqliteWorkspaceStore::open(&path) + let error = SqliteWorkspaceStore::open_for_test(&path) .err() .expect("migration must fail") .to_string(); @@ -17740,7 +18014,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx fn runtime_binding_identity_and_trust_uniqueness_are_workspace_scoped() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -17842,7 +18116,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert!( reopened .list_workspace_runtime_bindings("workspace-a", false) @@ -17892,7 +18166,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx fn workspace_runtime_connection_persists_without_authentication_evidence() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -17937,7 +18211,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); assert!(store.workspace_runtime_binding_matches(&persisted).unwrap()); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); assert_eq!( reopened .get_workspace_runtime_binding("workspace-a", "runtime-a") @@ -18191,7 +18465,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn runtime_binding_key_mutations_are_identity_guarded_idempotent_and_audited() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18329,7 +18603,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn embedded_runtime_binding_can_explicitly_rotate_restart_identity() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18429,7 +18703,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx conn.execute_batch("DROP TABLE typed_ticket_artifacts") .unwrap(); - let result = SqliteWorkspaceStore::from_connection(conn); + let result = SqliteWorkspaceStore::from_connection_for_test(conn); let error = match result { Ok(_) => panic!("schema drift unexpectedly passed startup verification"), Err(error) => error, @@ -18441,7 +18715,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn workspace_schema_rejects_cross_workspace_ticket_repository_reference_at_write_time() { let dir = tempfile::tempdir().unwrap(); let database_path = dir.path().join("workspace.sqlite"); - let store = SqliteWorkspaceStore::open(&database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); for workspace_id in ["workspace-a", "workspace-b"] { store .upsert_workspace(&WorkspaceRecord { @@ -18495,14 +18769,14 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(backend); - SqliteWorkspaceStore::open(&database_path).unwrap(); + SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); } #[tokio::test] async fn migrates_sqlite_and_preserves_workspace_record() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("control-plane.sqlite"); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); assert!( @@ -18520,7 +18794,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx }; store.upsert_workspace(&record).await.unwrap(); - let reopened = SqliteWorkspaceStore::open(&db).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!( reopened.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION @@ -18535,7 +18809,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn workspace_display_name_update_is_timestamp_guarded_and_preserves_identity() { let dir = tempfile::tempdir().unwrap(); let database_path = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); let record = WorkspaceRecord { workspace_id: "workspace-a".to_string(), owner_account_id: "owner-account".to_string(), @@ -18574,7 +18848,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx updated ); drop(store); - let reopened = SqliteWorkspaceStore::open(database_path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(database_path).unwrap(); assert_eq!( reopened .get_workspace(&record.workspace_id) @@ -18588,7 +18862,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn objective_creation_allocates_and_resolves_workspace_resource_key() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".into(), @@ -18634,7 +18908,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn worker_create_reservation_allocates_uuid_before_runtime_and_replays_exact_input() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -18938,7 +19212,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn worker_singleton_restart_retry_replays_exact_workdir_intent_and_rejects_changes() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -18998,7 +19272,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&database).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap(); let retried = reopened .reserve_worker_create( "workspace-a", @@ -19042,7 +19316,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn worker_singleton_ownership_is_workspace_scoped_durable_and_fences_old_generations() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); for workspace_id in ["workspace-a", "workspace-b"] { store .upsert_workspace(&WorkspaceRecord { @@ -19155,7 +19429,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); - let reopened = SqliteWorkspaceStore::open(database).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap(); assert_eq!( reopened .current_worker_singleton_owner("workspace-a", key) @@ -19168,7 +19442,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn concurrent_worker_singleton_reservations_have_one_winner() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19188,7 +19462,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let database = database.clone(); let barrier = barrier.clone(); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open(database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(database).unwrap(); let memory = store.get_workspace_memory_settings("workspace-a").unwrap(); barrier.wait(); store.reserve_worker_create( @@ -19208,7 +19482,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .collect::>(); assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); assert_eq!(results.iter().filter(|result| result.is_err()).count(), 1); - let store = SqliteWorkspaceStore::open(database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(database).unwrap(); let owner = store .current_worker_singleton_owner("workspace-a", "shared-key") .unwrap() @@ -19224,7 +19498,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn workspace_flow_sources_pin_content_and_builtins_stay_resources() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19317,7 +19591,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "returning to earlier content must reuse the immutable content" ); drop(store); - let reopened = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); assert_eq!( reopened .get_flow_source_content( @@ -19573,7 +19847,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx } ); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "case-workspace").unwrap(); store .with_conn(|conn| { @@ -19740,7 +20014,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); drop(backend); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); reopened .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -19807,7 +20081,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx before ); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "case-workspace").unwrap(); assert_eq!( backend @@ -19941,8 +20215,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("equivalence.db"); prepare_retained_schema(&path, 90); - let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); for object in [ "workspace_tags", "typed_ticket_tags", @@ -19995,7 +20269,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn schema_v91_case_constraint_verification_rejects_legacy_view_predicates() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store.with_conn(|conn| { conn.execute_batch("DROP VIEW ticket_active_worker_assignments; CREATE VIEW ticket_active_worker_assignments AS @@ -20156,7 +20430,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx // boundary under test. Targets need not be materialized by a Runtime. async fn manual_worker_start_store() -> (tempfile::TempDir, SqliteWorkspaceStore) { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -20244,7 +20518,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let threads = (0..2) .map(|index| { let store = - SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")) + .unwrap(); let barrier = barrier.clone(); std::thread::spawn(move || { barrier.wait(); @@ -20609,7 +20884,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .retain_ticket_claim_recovery("w", "claim", &fingerprint, &changed) .is_err() ); - let reopened = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + let reopened = + SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); assert_eq!( reopened.get_ticket_claim_recovery("w", "claim").unwrap(), Some(recovery) @@ -20667,7 +20943,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx store .rollback_ticket_assignment_operation("w", "claim") .unwrap(); - let reopened = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); + let reopened = + SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); let receipt = reopened .get_ticket_assignment_operation("w", "claim") .unwrap() @@ -20887,7 +21164,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx use ticket::TicketBackend; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -21015,7 +21292,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn ticket_worker_assignment_replaces_current_and_preserves_audit_history() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -21204,7 +21481,7 @@ INSERT INTO worker_registry ( ) .unwrap(); drop(store); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); let pending = store .get_ticket_assignment_operation("workspace-a", "reserved-operation") .unwrap() @@ -21328,7 +21605,7 @@ INSERT INTO worker_registry ( async fn role_assignment_routes_and_manual_start_are_state_fenced_atomically() { let dir = tempfile::tempdir().unwrap(); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-role".to_string(), @@ -21617,7 +21894,7 @@ INSERT INTO worker_registry ( fn startup_rejects_missing_workspace_resource_constraint_trigger() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch("DROP TRIGGER ticket_worker_assignments_validate_insert")?; @@ -21626,7 +21903,7 @@ INSERT INTO worker_registry ( .unwrap(); drop(store); - let error = match SqliteWorkspaceStore::open(&path) { + let error = match SqliteWorkspaceStore::open_for_test(&path) { Ok(_) => panic!("missing assignment constraint trigger must fail closed"), Err(error) => error, }; @@ -21642,7 +21919,7 @@ INSERT INTO worker_registry ( fn startup_accepts_retained_assignment_history_after_parent_cleanup() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -21706,7 +21983,7 @@ INSERT INTO worker_registry ( .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); reopened .with_conn(|conn| { let retained: i64 = conn.query_row( @@ -21734,7 +22011,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn workspace_bootstrap_validates_key_and_rejects_duplicate_workspace() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .upsert_account(&AccountRecord { account_id: "owner-account".to_string(), @@ -21866,7 +22143,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn repository_records_round_trip() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -21947,7 +22224,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22022,7 +22299,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn legacy_memory_reset_is_atomic_scoped_idempotent_and_exact() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); for (workspace_id, display_name) in [ ("local-dev", "Local Dev"), ("other-workspace", "Other Workspace"), @@ -22189,7 +22466,7 @@ INSERT INTO worker_registry ( async fn worker_workdir_registry_round_trips_and_preserves_pinned_retention() { let temp = tempfile::tempdir().unwrap(); let db = temp.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: "owner-account".to_string(), @@ -22558,7 +22835,7 @@ INSERT INTO worker_registry ( ); } drop(store); - let store = SqliteWorkspaceStore::open(&db).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!( store .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 0, None) @@ -22588,7 +22865,7 @@ INSERT INTO worker_registry ( let workdir_conflict = store.attach_worker_workdir(&workdir_conflict).unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open(&db).unwrap(); + let reopened = SqliteWorkspaceStore::open_for_test(&db).unwrap(); assert_eq!( reopened .list_worker_workdir_links("local-dev", &worker.worker) @@ -22606,7 +22883,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn worker_projection_preserves_equal_local_ids_from_distinct_runtimes() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22732,7 +23009,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn worker_projection_reconciles_snapshots_and_preserves_unavailable_lifecycle() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22907,7 +23184,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn late_runtime_event_does_not_recreate_retention_deleted_catalog_worker() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -23002,7 +23279,7 @@ INSERT INTO worker_registry ( async fn worker_control_grants_are_idempotent_scoped_and_support_internal_invalidation() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("control-grants.db"); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-control".to_string(), @@ -23072,7 +23349,7 @@ INSERT INTO worker_registry ( ); drop(store); - let store = SqliteWorkspaceStore::open(&database).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); assert_eq!( store .list_active_worker_control_grants( @@ -23130,7 +23407,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn account_and_login_records_round_trip() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { @@ -23489,9 +23766,9 @@ INSERT INTO worker_registry ( async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("resource-jobs.sqlite"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); seed_resource_job_workspace(&store).await; - let other = SqliteWorkspaceStore::open(&path).unwrap(); + let other = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let requests = [ resource_job_request("first", "subject-a", "candidate-1"), resource_job_request("followup", "subject-a", "candidate-2"), @@ -23607,7 +23884,7 @@ INSERT INTO worker_registry ( async fn backend_job_resource_retry_and_unknown_are_fenced_by_durable_cleanup() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("cleanup-jobs.sqlite"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); seed_resource_job_workspace(&store).await; let request = resource_job_request("failed", "subject-a", "candidate-1"); let first = store @@ -23713,7 +23990,7 @@ INSERT INTO worker_registry ( .unwrap(); finish_resource_job_cleanup(&store, &second); drop(store); - let restarted = SqliteWorkspaceStore::open(&path).unwrap(); + let restarted = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let active = restarted .find_active_backend_job_for_resource( "resource-workspace", @@ -23773,7 +24050,7 @@ INSERT INTO worker_registry ( async fn backend_job_resource_retry_races_new_intent_atomically_and_remains_bounded() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("retry-race.sqlite"); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); seed_resource_job_workspace(&store).await; // Generic resources use the same common reservation API without domain grants. let mut request = resource_job_request("generic", "subject-a", "candidate-1"); @@ -23796,7 +24073,7 @@ INSERT INTO worker_registry ( let mut followup = request.clone(); followup.job_id = "generic-followup".into(); followup.input = serde_json::json!({"candidate": "new"}); - let other = SqliteWorkspaceStore::open(&path).unwrap(); + let other = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let barrier = std::sync::Barrier::new(2); let (retry, reserve) = std::thread::scope(|scope| { let a = scope.spawn(|| { @@ -23882,7 +24159,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_all_older_attempt_cleanups_fence_retry_and_resource_release() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); seed_resource_job_workspace(&store).await; let mut request = resource_job_request("legacy-overlap", "subject-a", "candidate-1"); request.limits.max_attempts = 3; @@ -24006,7 +24283,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_empty_batch_surface_refresh_locks_subject_and_defers_candidates() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); seed_resource_job_workspace(&store).await; for surface_state in ["ungenerated", "stale", "failed"] { let subject = format!("subject-{surface_state}"); @@ -24126,7 +24403,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_completed_resource_releases_only_after_worker_cleanup() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); seed_resource_job_workspace(&store).await; let request = resource_job_request("completed", "subject-a", "candidate-1"); let first = store @@ -24180,7 +24457,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_jobs_fence_results_project_ownership_and_bound_retries() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); let workspace_id = "workspace-job"; store .upsert_account(&AccountRecord { diff --git a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs index 0ea9c64e7..69190eb1f 100644 --- a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs +++ b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs @@ -263,7 +263,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64 let tags_before = retained.as_ref().map(|_| classification_rows(&conn)); let auth_before = fixture_auth_rows(&conn); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?,93); assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11); @@ -490,7 +490,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history let metadata_before = stored_fixture_rows(&conn, "typed_ticket_case_metadata"); let auth_before = fixture_auth_rows(&conn); drop(conn); - let store = SqliteWorkspaceStore::open(&path).unwrap(); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?,93); diff --git a/crates/workspace-server/src/workdir_create_operations.rs b/crates/workspace-server/src/workdir_create_operations.rs index a41fc8ca7..a98284b1e 100644 --- a/crates/workspace-server/src/workdir_create_operations.rs +++ b/crates/workspace-server/src/workdir_create_operations.rs @@ -750,7 +750,7 @@ mod tests { #[test] fn retry_keeps_resolved_config_and_credential_candidates_after_fallback_moves() { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord { workspace_id: "workspace".to_string(), owner_account_id: "owner-account".to_string(), diff --git a/crates/workspace-server/src/workdir_removal.rs b/crates/workspace-server/src/workdir_removal.rs index 9c1f2a327..13b481fb4 100644 --- a/crates/workspace-server/src/workdir_removal.rs +++ b/crates/workspace-server/src/workdir_removal.rs @@ -925,7 +925,7 @@ mod tests { } async fn seeded_store() -> (SqliteWorkspaceStore, WorkdirRegistryRecord) { - let store = SqliteWorkspaceStore::in_memory().unwrap(); + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); store .upsert_account(&AccountRecord { account_id: "account-a".to_string(), diff --git a/crates/workspace-server/src/worker_projection.rs b/crates/workspace-server/src/worker_projection.rs index ed910de04..fc8140132 100644 --- a/crates/workspace-server/src/worker_projection.rs +++ b/crates/workspace-server/src/worker_projection.rs @@ -5,7 +5,7 @@ use chrono::Utc; use protocol::subscription::{ EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot, }; -use tokio::sync::broadcast; +use tokio::sync::{Semaphore, broadcast}; use tokio::task::JoinHandle; use crate::runtime_subscription::{ @@ -59,7 +59,14 @@ pub struct WorkerProjectionService { events: broadcast::Sender, ordering: Mutex<()>, attach_gate: tokio::sync::Mutex<()>, - tasks: Mutex>>, + tasks: Mutex>, +} + +struct RuntimeProjectionLifetime { + consumer: Option>, + // The permit belongs to the actual DB closure, not its cancellable async waiter. + // Keep this owner in the map even if attachment initialization is cancelled. + operations: Arc, } impl WorkerProjectionService { @@ -75,17 +82,89 @@ impl WorkerProjectionService { }) } + /// Stops consumers but does not claim that accepted database work has finished. + /// Async shutdown callers must use `shutdown_async` to drain accepted operations. pub fn shutdown(&self) { - for (_, task) in self + for lifetime in self .tasks .lock() .expect("Worker projection task lock poisoned") - .drain() + .values() { - task.abort(); + if let Some(task) = &lifetime.consumer { + task.abort(); + } } } + pub async fn shutdown_async(&self) { + let _attach = self.attach_gate.lock().await; + self.shutdown(); + let runtime_ids = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .keys() + .cloned() + .collect::>(); + for runtime_id in runtime_ids { + self.drain_runtime(&runtime_id).await; + } + self.tasks + .lock() + .expect("Worker projection task lock poisoned") + .clear(); + } + + async fn drain_runtime(&self, runtime_id: &str) { + let previous = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .get_mut(runtime_id) + .map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone())); + if let Some((consumer, operations)) = previous { + if let Some(consumer) = consumer { + consumer.abort(); + let _ = consumer.await; + } + // Aborting the waiter cannot abort a queued/running synchronous job. + // Cancellation here leaves the gate in the map for the next drain. + let _drained = operations + .acquire_owned() + .await + .expect("operation gate closed"); + } + } + + // Caller holds attach_gate. Install ownership before any initialization await. + async fn replace_runtime(&self, runtime_id: &str) -> Arc { + self.drain_runtime(runtime_id).await; + let operations = Arc::new(Semaphore::new(1)); + self.tasks + .lock() + .expect("Worker projection task lock poisoned") + .insert( + runtime_id.to_string(), + RuntimeProjectionLifetime { + consumer: None, + operations: operations.clone(), + }, + ); + operations + } + + pub async fn subscribe_ordered_async( + self: &Arc, + limit: usize, + ) -> crate::Result { + let service = self.clone(); + self.store + .db_executor() + .run(move || service.subscribe_ordered(limit)) + .await + } + pub fn subscribe_ordered(&self, limit: usize) -> crate::Result { let _ordering = self .ordering @@ -125,19 +204,10 @@ impl WorkerProjectionService { broker: &RuntimeSubscriptionBroker, runtime_id: &str, ) -> Result<(), RuntimeSubscriptionBrokerError> { - // Serialize replacement and wait for the old consumer to stop before the new - // lifetime can apply its snapshot. Late notifications can therefore never - // cross a Runtime ownership boundary. + // Serialize replacement, stop admission by aborting the consumer, and then + // drain ownership held by the actual SQL jobs before the new snapshot. let _attach = self.attach_gate.lock().await; - let previous = self - .tasks - .lock() - .expect("Worker projection task lock poisoned") - .remove(runtime_id); - if let Some(previous) = previous { - previous.abort(); - let _ = previous.await; - } + let operations = self.replace_runtime(runtime_id).await; let mut subscription = broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?; @@ -148,12 +218,25 @@ impl WorkerProjectionService { break; }; let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. }); - if let Err(error) = self.apply_broker_event(runtime_id, event) { + if let Err(error) = self + .apply_broker_event_owned_async(runtime_id, event, operations.clone()) + .await + { tracing::warn!( runtime_id, error = %error, "failed to initialize durable Worker projection; restarting from a fresh snapshot" ); + if matches!( + error, + crate::Error::DbOperationOutcomeUnknown + | crate::Error::DbExecutorUnavailable + | crate::Error::DbCapacityExceeded + ) { + // Do not automatically replay a possibly committed mutation or + // spin on executor overload; attachment must be retried explicitly. + return Err(RuntimeSubscriptionBrokerError::Closed); + } subscription.restart()?; continue; } @@ -169,12 +252,27 @@ impl WorkerProjectionService { let Some(service) = service.upgrade() else { return; }; - if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) { + if let Err(error) = service + .apply_broker_event_owned_async( + task_runtime_id.as_str(), + event, + operations.clone(), + ) + .await + { tracing::warn!( runtime_id = %task_runtime_id, error = %error, "failed to update durable Worker projection; restarting from a fresh snapshot" ); + if matches!( + error, + crate::Error::DbOperationOutcomeUnknown + | crate::Error::DbExecutorUnavailable + | crate::Error::DbCapacityExceeded + ) { + return; + } if subscription.restart().is_err() { return; } @@ -184,10 +282,61 @@ impl WorkerProjectionService { self.tasks .lock() .expect("Worker projection task lock poisoned") - .insert(runtime_id, task); + .get_mut(&runtime_id) + .expect("Runtime lifetime installed before initialization") + .consumer = Some(task); Ok(()) } + #[cfg(test)] + async fn seed_observation_async( + self: &Arc, + runtime_id: &str, + worker: &protocol::subscription::SubscriptionWorker, + observed_at: &str, + ) -> crate::Result<()> { + let service = self.clone(); + let runtime_id = runtime_id.to_string(); + let worker = worker.clone(); + let observed_at = observed_at.to_string(); + self.store + .db_executor() + .run(move || service.seed_observation(&runtime_id, &worker, &observed_at)) + .await + } + + pub async fn refresh_async(self: &Arc, worker: &RuntimeWorkerRef) -> crate::Result<()> { + let service = self.clone(); + let worker = worker.clone(); + self.store + .db_executor() + .run(move || service.refresh(&worker)) + .await + } + + async fn apply_broker_event_owned_async( + self: &Arc, + runtime_id: &str, + event: BrokerSubscriptionEvent, + operations: Arc, + ) -> crate::Result<()> { + let operation = operations + .acquire_owned() + .await + .expect("operation gate closed"); + let service = self.clone(); + let runtime_id = runtime_id.to_string(); + self.store + .db_executor() + .run(move || { + // The permit must outlive SQL commit AND ordered publication, even + // when the executor result receiver has been dropped by cancellation. + let _operation = operation; + service.apply_broker_event(&runtime_id, event) + }) + .await + } + pub fn seed_observation( &self, runtime_id: &str, @@ -313,7 +462,11 @@ mod tests { tempfile::TempDir, ) { let temp = tempfile::tempdir().unwrap(); - let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap()); + let store = Arc::new( + SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")) + .unwrap() + .with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()), + ); store .upsert_account(&AccountRecord { account_id: "owner".to_string(), @@ -355,6 +508,266 @@ mod tests { } } + fn observation( + state: protocol::subscription::SubscriptionWorkerState, + ) -> protocol::subscription::SubscriptionWorker { + protocol::subscription::SubscriptionWorker { + restore_observation_token: None, + worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(), + runtime_id: Some("runtime-a".to_string()), + resource_key: None, + availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, + worker_state: None, + state, + has_running_internal_workers: false, + workspace_id: Some(WORKSPACE_ID.to_string()), + display_name: None, + profile: None, + job: None, + workdir_attachments: Vec::new(), + } + } + + // Real second-connection contention; the controller releases automatically on + // failure, so a regression that blocks the Tokio thread cannot hang the suite. + struct WriteLock { + release: Option>, + controller: Option>, + } + + impl WriteLock { + fn acquire(path: std::path::PathBuf) -> Self { + let (release, released) = std::sync::mpsc::channel(); + let (locked, ready) = std::sync::mpsc::channel(); + let controller = std::thread::spawn(move || { + let connection = rusqlite::Connection::open(path).unwrap(); + connection.execute_batch("BEGIN IMMEDIATE").unwrap(); + locked.send(()).unwrap(); + let _ = released.recv_timeout(std::time::Duration::from_secs(3)); + connection.execute_batch("ROLLBACK").unwrap(); + }); + ready.recv().unwrap(); + Self { + release: Some(release), + controller: Some(controller), + } + } + + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + } + } + + impl Drop for WriteLock { + fn drop(&mut self) { + self.release(); + if let Some(controller) = self.controller.take() { + controller.join().unwrap(); + } + } + } + + async fn wait_for_sql(service: &WorkerProjectionService) { + tokio::time::timeout(std::time::Duration::from_secs(1), async { + // With the second connection holding the write lock, a held ordering + // gate identifies the accepted synchronous mutation, not a fake future. + while service.ordering.try_lock().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("projection job did not enter synchronous SQL"); + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload() + { + use protocol::subscription::SubscriptionWorkerState; + let (store, service, temp) = fixture().await; + store + .upsert_worker_registry(&worker("runtime-a", "worker-a")) + .unwrap(); + let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); + assert_eq!(subscription.take_snapshot().len(), 1); + let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); + let publishing_service = service.clone(); + let publication = tokio::spawn(async move { + publishing_service + .seed_observation_async( + "runtime-a", + &observation(SubscriptionWorkerState::Idle), + "2", + ) + .await + }); + wait_for_sql(&service).await; + let started = std::time::Instant::now(); + let (control, controlled) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + control.send("control progressed").unwrap(); + }); + let (_, control_result) = tokio::join!( + tokio::time::sleep(std::time::Duration::from_millis(20)), + controlled, + ); + assert_eq!(control_result.unwrap(), "control progressed"); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "SQLite blocked the current-thread executor" + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + lock.release(); + publication.await.unwrap().unwrap(); + let committed = subscription.recv().await.unwrap(); + let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); + let snapshot = snapshot.take_snapshot(); + assert_eq!( + committed.changes, + vec![WorkerCatalogChange::Upsert(snapshot[0].clone())] + ); + assert_eq!( + snapshot[0].observation.as_ref().unwrap().worker.state, + SubscriptionWorkerState::Idle + ); + service + .seed_observation_async( + "runtime-a", + &observation(SubscriptionWorkerState::Running), + "3", + ) + .await + .unwrap(); + // Delayed readers receive the first transaction's payload, not a re-query + // of mutable authority after a later commit. + let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else { + panic!("expected upsert"); + }; + assert_eq!( + first.observation.as_ref().unwrap().worker.state, + SubscriptionWorkerState::Idle + ); + service + .refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a")) + .await + .unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() { + use crate::runtime_subscription::BrokerSubscriptionEvent; + use protocol::subscription::SubscriptionWorkerState; + let (store, service, temp) = fixture().await; + store + .upsert_worker_registry(&worker("runtime-a", "worker-a")) + .unwrap(); + let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); + subscription.take_snapshot(); + let operations = service.replace_runtime("runtime-a").await; + let old_operations = operations.clone(); + let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); + let old_service = service.clone(); + let consumer = tokio::spawn(async move { + old_service + .apply_broker_event_owned_async( + "runtime-a", + BrokerSubscriptionEvent::Event { + payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted { + worker: observation(SubscriptionWorkerState::Running), + }, + }, + operations, + ) + .await + .unwrap(); + }); + service + .tasks + .lock() + .unwrap() + .get_mut("runtime-a") + .unwrap() + .consumer = Some(consumer); + wait_for_sql(&service).await; + let replacing_service = service.clone(); + let mut replacement = tokio::spawn(async move { + let _attach = replacing_service.attach_gate.lock().await; + replacing_service.replace_runtime("runtime-a").await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) + .await + .is_err() + ); + replacement.abort(); + assert!(replacement.await.unwrap_err().is_cancelled()); + assert_eq!( + old_operations.available_permits(), + 0, + "cancelled drain lost accepted SQL ownership" + ); + // A second replacement must still observe the old gate after cancellation. + let replacing_service = service.clone(); + let mut replacement = tokio::spawn(async move { + let _attach = replacing_service.attach_gate.lock().await; + let operations = replacing_service.replace_runtime("runtime-a").await; + replacing_service + .apply_broker_event_owned_async( + "runtime-a", + BrokerSubscriptionEvent::Snapshot { + snapshot: protocol::subscription::SubscriptionSnapshot::Workers { + workers: vec![observation(SubscriptionWorkerState::Idle)], + }, + }, + operations, + ) + .await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) + .await + .is_err() + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + lock.release(); + replacement.await.unwrap().unwrap(); + assert_eq!(old_operations.available_permits(), 1); + let old_commit = subscription.recv().await.unwrap(); + let new_commit = subscription.recv().await.unwrap(); + for (commit, expected) in [ + (old_commit, SubscriptionWorkerState::Running), + (new_commit, SubscriptionWorkerState::Idle), + ] { + let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else { + panic!("expected one committed upsert"); + }; + assert_eq!(record.observation.as_ref().unwrap().worker.state, expected); + } + let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); + assert_eq!( + snapshot.take_snapshot()[0] + .observation + .as_ref() + .unwrap() + .worker + .state, + SubscriptionWorkerState::Idle + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + service.shutdown_async().await; + assert!(service.tasks.lock().unwrap().is_empty()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn snapshot_registration_is_atomic_with_concurrent_publication() { let (store, service, _temp) = fixture().await; diff --git a/crates/workspace-server/src/worker_source.rs b/crates/workspace-server/src/worker_source.rs index 3d51476d1..3edbb5d3e 100644 --- a/crates/workspace-server/src/worker_source.rs +++ b/crates/workspace-server/src/worker_source.rs @@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError { Replay, #[error("source Worker is not a current member of this Workspace Runtime catalog")] WorkerCatalogMembership, + #[error("source proof authority is unavailable")] + Unavailable, #[error("source proof authority failed: {0}")] Authority(String), } +impl From for WorkerMutationSourceProofError { + fn from(error: crate::Error) -> Self { + match error { + crate::Error::DbCapacityExceeded + | crate::Error::DbExecutorUnavailable + | crate::Error::DbOperationOutcomeUnknown + | crate::Error::Sqlite(_) => Self::Unavailable, + other => Self::Authority(other.to_string()), + } + } +} + pub async fn verify_worker_remove_source( api: &WorkspaceApi, proof: PresentedWorkerMutationSourceProof<'_>, @@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with( let trusted = store .get_workspace_runtime_binding(&config.workspace_id, &unverified.iss) .await - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))? + .map_err(WorkerMutationSourceProofError::from)? .filter(|record| record.revoked_at.is_none()) .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?; let public_key = trusted.public_key.as_str(); @@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with( runtime_id: claims.iss.clone(), worker_id: claims.worker_id.clone(), }; - let member = store - .get_worker_registry(&config.workspace_id, &worker) - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; - if member.is_none() { - return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); - } + let workspace_id = config.workspace_id.clone(); + let catalog_store = Arc::clone(store); store - .require_current_worker_singleton_owner(&config.workspace_id, &worker) - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + .db_executor() + .run(move || { + let member = catalog_store + .get_worker_registry(&workspace_id, &worker) + .map_err(WorkerMutationSourceProofError::from)?; + if member.is_none() { + return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); + } + catalog_store + .require_current_worker_singleton_owner(&workspace_id, &worker) + .map_err(WorkerMutationSourceProofError::from)?; + Ok(()) + }) + .await?; let consumed_at = chrono::Utc::now().to_rfc3339(); let consumed = store @@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with( &consumed_at, ) .await - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + .map_err(WorkerMutationSourceProofError::from)?; if !consumed { return Err(WorkerMutationSourceProofError::Replay); } diff --git a/crates/workspace-server/src/worker_source/runtime_request.rs b/crates/workspace-server/src/worker_source/runtime_request.rs index 9e79fd3b1..b8c2e7226 100644 --- a/crates/workspace-server/src/worker_source/runtime_request.rs +++ b/crates/workspace-server/src/worker_source/runtime_request.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use worker_runtime::auth::{ RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims, RuntimeRequestSourceExpectation, decode_runtime_request_source_claims, @@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError { pub(crate) exp: Option, pub(crate) now: Option, pub(crate) time_delta_seconds: Option, + /// Operational unavailability is not a proof or principal rejection. + pub(crate) availability: bool, } impl RuntimeRequestProofError { + pub fn is_availability_error(&self) -> bool { + self.availability + } + + fn unavailable(mut self, reason: &'static str) -> Self { + self.reason = reason; + self.availability = true; + self + } + fn undecoded() -> Self { Self { reason: "invalid", @@ -47,6 +61,7 @@ impl RuntimeRequestProofError { exp: None, now: None, time_delta_seconds: None, + availability: false, } } @@ -106,6 +121,18 @@ impl RuntimeRequestProofError { } } +impl From for RuntimeRequestProofError { + fn from(error: crate::Error) -> Self { + let reason = match error { + crate::Error::DbCapacityExceeded => "db_capacity_exceeded", + crate::Error::DbExecutorUnavailable => "db_executor_unavailable", + crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown", + _ => "database_authority_unavailable", + }; + Self::undecoded().unavailable(reason) + } +} + /// Domain-separated fixed-length fingerprints, not raw request-derived IDs. /// Operators can correlate a known ID by applying this same function. pub fn diagnostic_id_hash(id: &str) -> String { @@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof( body_digest: &str, ) -> Result { verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + Arc::clone(&api.store), &api.config, proof, workspace_id, @@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof( #[allow(clippy::too_many_arguments)] pub async fn verify_runtime_request_source_proof_with_store( - store: &dyn ControlPlaneStore, + store: Arc, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store( #[allow(clippy::too_many_arguments)] pub(crate) async fn verify_runtime_request_source_proof_with_clock( - store: &dyn ControlPlaneStore, + store: Arc, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( let unverified = decode_runtime_request_source_claims(proof) .map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?; let diagnostic = RuntimeRequestProofError::decoded(&unverified); - let audience = remote_audience(config, workspace_id) - .map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?; + let audience = remote_audience(config, workspace_id).map_err(|_| { + diagnostic + .clone() + .unavailable("backend_audience_unavailable") + })?; let trusted = store .get_workspace_runtime_binding(workspace_id, &unverified.iss) .await .map_err(|_| { diagnostic .clone() - .reason("runtime_trust_authority_unavailable") + .unavailable("runtime_trust_authority_unavailable") })? .filter(|record| record.revoked_at.is_none()) .ok_or_else(|| { @@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( &consumed_at, ) .await - .map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))? + .map_err(|_| { + diagnostic + .clone() + .unavailable("replay_authority_unavailable") + })? { return Err(diagnostic.reason("replay")); } @@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( runtime_id: claims.iss.clone(), worker_id: worker_id.to_owned(), }; - let member = store - .get_worker_registry(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_catalog_authority_unavailable") - })?; - let reserved = store - .has_active_worker_create_reservation(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_reservation_authority_unavailable") - })?; - if member.is_none() && !reserved { - return Err(diagnostic.reason("worker_catalog_membership")); - } - store - .require_current_worker_singleton_owner(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_singleton_authority_denied") + let workspace_id = workspace_id.to_owned(); + let executor = store.db_executor(); + let stage_diagnostic = diagnostic.clone(); + // Only synchronous catalog authority lives in this closure. The trust + // and JTI async wrappers above already offload through the executor. + executor + .run(move || { + let member = store + .get_worker_registry(&workspace_id, &worker) + .map_err(|_| { + stage_diagnostic + .clone() + .unavailable("worker_catalog_authority_unavailable") + })?; + let reserved = store + .has_active_worker_create_reservation(&workspace_id, &worker) + .map_err(|_| { + stage_diagnostic + .clone() + .unavailable("worker_reservation_authority_unavailable") + })?; + if member.is_none() && !reserved { + return Err(stage_diagnostic.reason("worker_catalog_membership")); + } + store + .require_current_worker_singleton_owner(&workspace_id, &worker) + .map_err(|error| { + if matches!( + error, + crate::Error::DbCapacityExceeded + | crate::Error::DbExecutorUnavailable + | crate::Error::DbOperationOutcomeUnknown + | crate::Error::Sqlite(_) + ) { + stage_diagnostic + .clone() + .unavailable("worker_singleton_authority_unavailable") + } else { + stage_diagnostic + .clone() + .reason("worker_singleton_authority_denied") + } + })?; + Ok(()) + }) + .await + .map_err(|error: RuntimeRequestProofError| { + if error.verification == "unverified" && error.is_availability_error() { + diagnostic.clone().unavailable(error.reason) + } else { + error + } })?; } Ok(VerifiedRuntimeRequestSource { @@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( authentication_binding_id: trusted.binding_id, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::{ + AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode, + WorkspaceRuntimeBinding, + }; + use std::time::Duration; + use worker_runtime::auth::{ + BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner, + request_body_digest, + }; + + const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa"; + const NOW: i64 = 1_700_000_000; + const PATH: &str = "/api/runtime/v1/resources/fetch"; + + // rusqlite's busy handler accepts a function pointer, not a closure. Each + // fixture owns a dedicated single-worker executor, so this thread-local + // handshake is isolated from all parallel tests and product connections. + thread_local! { + static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell, + std::sync::mpsc::Receiver<()>, + )>> = const { std::cell::RefCell::new(None) }; + } + + fn acknowledge_sqlite_contention(_attempt: i32) -> bool { + SQLITE_BUSY_HANDSHAKE.with(|slot| { + let Some((blocked, retry)) = slot.borrow_mut().take() else { + return false; + }; + let _ = blocked.send(()); + // The test releases SQLite's writer lock before allowing retry. + // Dropping the sender on panic also releases this worker promptly. + retry.recv_timeout(Duration::from_secs(3)).is_ok() + }) + } + + async fn fixture( + root: &std::path::Path, + ) -> (Arc, ServerConfig, String, String) { + let db_path = root.join("auth.db"); + let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::open_for_test(&db_path) + .unwrap() + .with_db_executor(executor), + ); + store + .upsert_account(&AccountRecord { + account_id: "account-auth".into(), + kind: "user".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + let workspace = WorkspaceRecord { + workspace_id: WORKSPACE.into(), + owner_account_id: "account-auth".into(), + display_name: "Auth".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }; + store.upsert_workspace(&workspace).await.unwrap(); + let config = ServerConfig { + workspace_id: WORKSPACE.into(), + workspace_display_name: "Auth".into(), + workspace_created_at: "1".into(), + workspace_execution_root: root.to_owned(), + database_path: db_path, + data_root: root.to_owned(), + embedded_runtime_store_root: root.join("runtime"), + static_assets_dir: None, + auth: crate::server::AuthConfig::Passkey { + rp_id: "localhost".into(), + origin: "http://localhost".into(), + public_base_url: "http://localhost".into(), + cookie_name: "session".into(), + }, + max_records: 10, + repositories: vec![], + runtime_event_sources: vec![], + remote_runtime_sources: vec![], + backend_base_url: Some("server-auth".into()), + }; + let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap(); + store + .upsert_workspace_runtime_binding_record( + WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: "runtime-auth".into(), + display_name: "Auth".into(), + base_url: "http://127.0.0.1:1".into(), + public_key: identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer, + created_at: "1".into(), + updated_at: "1".into(), + revoked_at: None, + }, + false, + ) + .await + .unwrap(); + let binding = ControlPlaneStore::get_workspace_runtime_binding( + store.as_ref(), + WORKSPACE, + "runtime-auth", + ) + .await + .unwrap() + .unwrap(); + let proof = RuntimeRequestSourceSigner::from_identity(&identity) + .issue( + "server-auth", + WORKSPACE, + None, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + b"{}", + NOW, + 30, + ) + .unwrap(); + (store, config, proof, binding.binding_id) + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() { + let root = tempfile::tempdir().unwrap(); + let (store, config, proof, binding_id) = fixture(root.path()).await; + // A real independent SQLite writer prevents JTI insertion while trust + // reads remain possible. Rollback on Drop also releases this on panic. + let mut writer = rusqlite::Connection::open(&config.database_path).unwrap(); + let lock = writer + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .unwrap(); + let (blocked, contention_reached) = tokio::sync::oneshot::channel(); + let (retry, retry_rx) = std::sync::mpsc::channel(); + let setup_store = store.clone(); + store + .db_executor() + .run(move || -> crate::Result<()> { + SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx))); + setup_store.with_conn(|conn| { + conn.busy_handler(Some(acknowledge_sqlite_contention))?; + Ok(()) + }) + }) + .await + .unwrap(); + let request_store = store.clone(); + let request_config = config.clone(); + let request_proof = proof.clone(); + let task = tokio::spawn(async move { + verify_runtime_request_source_proof_with_clock( + request_store, + &request_config, + &request_proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(1), contention_reached) + .await + .expect( + "SQLite JTI persistence must report real writer contention without blocking Tokio", + ) + .unwrap(); + // The busy callback acknowledges an actual SQLite write blocked by the + // independent writer, then holds retry until explicitly released. + assert!( + !task.is_finished(), + "JTI must remain pending while SQLite writer is locked" + ); + let control = tokio::spawn(async { "control-responsive" }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), control) + .await + .unwrap() + .unwrap(), + "control-responsive" + ); + // This timer is the behavior being tested, not a guessed wait for SQL. + tokio::time::timeout( + Duration::from_secs(1), + tokio::time::sleep(Duration::from_millis(1)), + ) + .await + .unwrap(); + assert!( + !task.is_finished(), + "timer must advance while JTI remains blocked" + ); + lock.rollback().unwrap(); + retry.send(()).unwrap(); + let source = tokio::time::timeout(Duration::from_secs(3), task) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(source.runtime_id, "runtime-auth"); + assert_eq!(source.authentication_binding_id, binding_id); + let replay = verify_runtime_request_source_proof_with_clock( + store, + &config, + &proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ) + .await + .unwrap_err(); + assert_eq!(replay.reason, "replay"); + assert!(!replay.is_availability_error()); + } + + #[tokio::test] + async fn saturated_database_is_availability_failure_not_invalid_proof() { + let root = tempfile::tempdir().unwrap(); + let (store, config, proof, _) = fixture(root.path()).await; + // Fill the isolated executor with a blocked job and queued jobs. No + // scheduling sleeps: the worker explicitly acknowledges its start. + let executor = store.db_executor(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let blocker_executor = executor.clone(); + let blocker = tokio::spawn(async move { + blocker_executor + .run(move || -> crate::Result<()> { + let _ = started.send(()); + release_rx.recv().unwrap(); + Ok(()) + }) + .await + }); + tokio::time::timeout(Duration::from_secs(1), started_rx) + .await + .unwrap() + .unwrap(); + let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) })); + assert!(futures::poll!(queued.as_mut()).is_pending()); + let outcome = tokio::time::timeout( + Duration::from_secs(1), + verify_runtime_request_source_proof_with_clock( + store, + &config, + &proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ), + ) + .await; + // Release before assertions so a failed contract never strands a worker. + release.send(()).unwrap(); + blocker.await.unwrap().unwrap(); + queued.await.unwrap(); + let error = outcome + .expect("capacity rejection must not wait for queued SQL") + .unwrap_err(); + assert_eq!(error.reason, "runtime_trust_authority_unavailable"); + assert!(error.is_availability_error()); + } +} diff --git a/crates/workspace-server/src/workspace_catalog.rs b/crates/workspace-server/src/workspace_catalog.rs index 5d9679d9f..78ce46f45 100644 --- a/crates/workspace-server/src/workspace_catalog.rs +++ b/crates/workspace-server/src/workspace_catalog.rs @@ -285,7 +285,7 @@ mod tests { #[tokio::test] async fn create_is_atomic_and_exact_retries_converge() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let service = WorkspaceCatalogService::new( store.clone(), Arc::new( @@ -339,7 +339,7 @@ mod tests { #[test] fn create_rejects_noncanonical_repository_keys_without_persisting_a_workspace() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let service = WorkspaceCatalogService::new( store.clone(), Arc::new( @@ -378,7 +378,7 @@ mod tests { for with_repository in [false, true] { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let owner_account_id = owner_account(store.as_ref()); let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default()); let service = WorkspaceCatalogService::new( @@ -418,7 +418,7 @@ mod tests { drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let restarted = WorkspaceCatalogService::new(store.clone(), materials); let created = restarted.create(request, owner_account_id).unwrap(); let identity = store @@ -435,7 +435,7 @@ mod tests { for with_repository in [false, true] { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let owner_account_id = owner_account(store.as_ref()); let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default()); let service = WorkspaceCatalogService::new(store.clone(), materials.clone()); @@ -486,7 +486,7 @@ mod tests { drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let restarted = WorkspaceCatalogService::new(store.clone(), materials); let created = restarted.create(request, owner_account_id).unwrap(); let identity = store @@ -500,7 +500,7 @@ mod tests { #[tokio::test] async fn idempotency_key_reuse_with_different_payload_is_rejected() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner_account_id = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store, @@ -532,7 +532,7 @@ mod tests { #[test] fn repository_free_create_initializes_authority_and_concurrent_retries_converge() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), @@ -611,7 +611,7 @@ mod tests { #[test] fn replay_reads_current_repository_at_initial_key_and_never_recreates_it() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), @@ -673,13 +673,13 @@ mod tests { repository: None, }; let (owner, created) = { - let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new(store, materials.clone()); let created = service.create(request.clone(), owner.clone()).unwrap(); (owner, created) }; - let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); let reopened = WorkspaceCatalogService::new(store.clone(), materials); let replay = reopened.create(request, owner).unwrap(); assert_eq!(replay.workspace, created.workspace); @@ -739,7 +739,7 @@ mod tests { #[test] fn create_rejects_non_user_account_owners() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); store .upsert_account(&AccountRecord { account_id: "organization-owner".to_string(), @@ -777,7 +777,7 @@ mod tests { #[test] fn catalog_list_is_scoped_to_the_required_owner_account() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner_a = owner_account(store.as_ref()); let owner_b = "account-owner-b".to_string(); store @@ -845,7 +845,7 @@ mod tests { #[test] fn remote_repository_creation_persists_typed_source_without_auth_metadata() { - let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); let owner_account_id = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index e241f8aee..5003d4153 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -236,7 +236,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { "Workspace deletion request exceeds bounded field limits".to_string(), )); } - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { if let Some(existing) = read_operation(tx, &request.operation_id)? { if existing.actor_account_id != actor_account_id { return Err(Error::WorkspacePermissionDenied( @@ -369,7 +369,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { child_operation_id: &str, ) -> Result { validate_operation_id(child_operation_id)?; - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { let operation = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? .response; @@ -411,7 +411,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { failure_category: Option<&str>, ) -> Result { validate_operation_projection(child_operation_ids, blockers)?; - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { let current = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? .response; @@ -462,7 +462,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { &self, operation_id: &str, ) -> Result { - self.with_transaction(|tx| { + self.with_immediate_transaction(|tx| { tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?; let operation = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; @@ -922,7 +922,7 @@ mod tests { use tempfile::tempdir; fn setup() -> (SqliteWorkspaceStore, String, String) { - seed_workspaces(SqliteWorkspaceStore::in_memory().expect("store")) + seed_workspaces(SqliteWorkspaceStore::in_memory_for_test().expect("store")) } fn seed_workspaces(store: SqliteWorkspaceStore) -> (SqliteWorkspaceStore, String, String) { @@ -989,7 +989,7 @@ mod tests { let dir = tempdir().expect("tempdir"); let path = dir.path().join("server.db"); let (store, owner, workspace_id) = - seed_workspaces(SqliteWorkspaceStore::open(&path).expect("store")); + seed_workspaces(SqliteWorkspaceStore::open_for_test(&path).expect("store")); for workspace in ["workspace-a", "workspace-b"] { let backend = ticket::SqliteTicketBackend::open_verified(&path, workspace).unwrap(); let initial = backend @@ -1019,7 +1019,7 @@ mod tests { // Seed domain handling evidence under a transaction. This fixture // exercises owner purge, not singleton authentication or delivery. store - .with_transaction(|tx| { + .with_immediate_transaction(|tx| { ticket::sqlite_update_ticket_assignment_tracking_in_transaction( tx, workspace, diff --git a/crates/workspace-server/src/workspace_drive_host.rs b/crates/workspace-server/src/workspace_drive_host.rs index 410c0c049..bc0bfc390 100644 --- a/crates/workspace-server/src/workspace_drive_host.rs +++ b/crates/workspace-server/src/workspace_drive_host.rs @@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost { impl WorkspaceDriveHost { pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result { let storage = storage.clone(); - let registration = tokio::task::spawn_blocking(move || Drive::register(&storage)) - .await - .map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))? - .map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?; + let registration = crate::db_executor::DbExecutor::shared() + .run(move || { + Drive::register(&storage).map_err(|error| { + Error::Store(format!("failed to register Drive storage: {error}")) + }) + }) + .await?; Ok(Self { registration, purged: AtomicBool::new(false), @@ -80,21 +83,21 @@ impl WorkspaceDriveHost { } let storage = storage.clone(); let registration = self.registration.clone(); - let drive = tokio::task::spawn_blocking(move || { - reject_symlink_ancestors(&root)?; - // Domain IMMEDIATE transactions lock the attached server authority - // as well as Drive metadata. Deletion reservation and cached-handle - // operations therefore serialize with other server DB writes. - Drive::open_with_workspace_authority( - &storage, - ®istration, - &root, - &authority_database, - ) - .map_err(Error::from) - }) - .await - .map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??; + let drive = crate::db_executor::DbExecutor::shared() + .run(move || { + reject_symlink_ancestors(&root)?; + // Domain IMMEDIATE transactions lock the attached server authority + // as well as Drive metadata. Deletion reservation and cached-handle + // operations therefore serialize with other server DB writes. + Drive::open_with_workspace_authority( + &storage, + ®istration, + &root, + &authority_database, + ) + .map_err(Error::from) + }) + .await?; *cached = Some(drive.clone()); Ok(drive) } @@ -109,9 +112,9 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root, authority_database).await?; - tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from)) + crate::db_executor::DbExecutor::shared() + .run(move || drive.fence().map_err(Error::from)) .await - .map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))? } pub(crate) async fn purge( @@ -124,20 +127,20 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root.clone(), authority_database).await?; - tokio::task::spawn_blocking(move || -> Result<()> { - reject_symlink_ancestors(&root)?; - // Domain purge persists its tombstone before removing blobs/nodes/receipts. - drive.purge()?; - // Remove the empty blob root, never the data root or another Workspace. - // Missing roots are successful retries; other failures retain FeatureStorage. - match std::fs::remove_dir_all(&root) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - } - }) - .await - .map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??; + crate::db_executor::DbExecutor::shared() + .run(move || -> Result<()> { + reject_symlink_ancestors(&root)?; + // Domain purge persists its tombstone before removing blobs/nodes/receipts. + drive.purge()?; + // Remove the empty blob root, never the data root or another Workspace. + // Missing roots are successful retries; other failures retain FeatureStorage. + match std::fs::remove_dir_all(&root) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + }) + .await?; // A later signing-material/finalization failure can retry after the // feature DB has been closed and removed by FeatureStorage.delete. self.purged.store(true, Ordering::Release); diff --git a/crates/workspace-server/src/workspace_signing_identity.rs b/crates/workspace-server/src/workspace_signing_identity.rs index f724f2ca1..959f28805 100644 --- a/crates/workspace-server/src/workspace_signing_identity.rs +++ b/crates/workspace-server/src/workspace_signing_identity.rs @@ -780,7 +780,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); store .upsert_account(&AccountRecord { account_id: "account-1".to_string(), @@ -833,7 +833,7 @@ mod tests { drop(failing_service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let service = WorkspaceSigningIdentityService::new(store.clone(), materials.clone()); let provisioned = service .provision_existing("workspace-1", "account-1") @@ -962,7 +962,7 @@ mod tests { ); drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); let restarted = WorkspaceSigningIdentityService::new(store.clone(), materials.clone()); let recovered = restarted .provision_existing("workspace-2", "account-1") diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index 8b0e0b818..a62faa34e 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription( worker_id, runtime_id: Some(runtime_id), } => { - let connection = match WorkspaceWorker::resolve( + let connection = match WorkspaceWorker::resolve_async( &api, &runtime_id, worker_id.as_str(), - ) { + ) + .await + { Ok(worker) => worker .connect_protocol(&context) .await @@ -366,7 +368,8 @@ async fn run_workspace_workers( ) { let Ok(mut projection) = api .worker_projection - .subscribe_ordered(api.config.max_records.max(1)) + .subscribe_ordered_async(api.config.max_records.max(1)) + .await else { let _ = send_rejected( &outbound, @@ -377,11 +380,22 @@ async fn run_workspace_workers( .await; return; }; - let mut workers = projection - .take_snapshot() - .into_iter() - .filter_map(|record| project_registry_worker(&api, record)) - .collect::>(); + let Ok(mut workers) = project_registry_workers_async( + api.store.clone(), + api.config.workspace_id.clone(), + projection.take_snapshot(), + ) + .await + else { + let _ = send_rejected( + &outbound, + request_id, + SubscriptionRejectionCode::Internal, + "failed to enrich durable Worker projection".to_string(), + ) + .await; + return; + }; sort_workers(&mut workers); if send_frame( &outbound, @@ -406,7 +420,29 @@ async fn run_workspace_workers( for change in event.changes { let payload = match change { crate::worker_projection::WorkerProjectionChange::Upsert(record) => { - let Some(worker) = project_registry_worker(&api, record) else { + let workers = project_registry_workers_async( + api.store.clone(), + api.config.workspace_id.clone(), + vec![record], + ) + .await; + let Ok(mut workers) = workers else { + // Admission failure cannot silently lose a live + // update: terminate and require a fresh snapshot. + let _ = send_frame( + &outbound, + SubscriptionFrame::new(SubscriptionFramePayload::Event( + SubscriptionEvent::SubscriptionClosed { + subscription_id: subscription_id.clone(), + code: SubscriptionTerminationCode::Lagged, + message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(), + }, + )), + ) + .await; + return; + }; + let Some(worker) = workers.pop() else { continue; }; SubscriptionEventPayload::WorkerUpserted { worker } @@ -450,8 +486,28 @@ async fn run_workspace_workers( } } +async fn project_registry_workers_async( + store: std::sync::Arc, + workspace_id: String, + records: Vec, +) -> crate::Result> { + store + .db_executor() + .run(move || { + // Keep the committed Worker payload; only the existing Workspace-scoped + // workdir/history/repository enrichment reads occur inside this job. + Ok(records + .into_iter() + .filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record)) + .collect()) + }) + .await +} + +// Synchronous helpers stay synchronous; they run only inside the job above. fn project_registry_worker( - api: &WorkspaceApi, + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, record: crate::store::WorkerRegistryProjectionRecord, ) -> Option { let runtime_id = record.registry.worker.runtime_id.clone(); @@ -491,29 +547,25 @@ fn project_registry_worker( attempt_id: job.attempt_id, purpose: job.purpose, }); - if !project_working_directory(api, &record.registry.worker, &mut worker) { + if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) { return None; } Some(worker) } fn project_working_directory( - api: &WorkspaceApi, + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, worker_ref: &RuntimeWorkerRef, worker: &mut SubscriptionWorker, ) -> bool { - let Ok(links) = api - .store - .list_worker_workdir_links(&api.config.workspace_id, worker_ref) - else { + let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else { return false; }; if !links.is_empty() { let mut attachments = Vec::with_capacity(links.len()); for link in links { - let Ok(Some(workdir)) = api - .store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id) else { return false; }; @@ -533,9 +585,8 @@ fn project_working_directory( } worker.workdir_attachments = attachments; } else { - let Ok(has_link_history) = api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref) + let Ok(has_link_history) = + store.worker_workdir_link_history_exists(workspace_id, worker_ref) else { return false; }; @@ -543,13 +594,17 @@ fn project_working_directory( worker.workdir_attachments.clear(); } } - project_repository_keys(api, worker) + project_repository_keys(store, workspace_id, worker) } -fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool { +fn project_repository_keys( + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, + worker: &mut SubscriptionWorker, +) -> bool { project_repository_keys_with(worker, |repository_id| { - api.store - .get_repository(&api.config.workspace_id, repository_id) + store + .get_repository(workspace_id, repository_id) .ok() .flatten() .map(|repository| repository.repository_key) @@ -634,9 +689,8 @@ mod tests { protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap() } - #[test] - fn external_workdir_attachment_does_not_remove_workspace_worker() { - let mut worker = SubscriptionWorker { + fn external_worker() -> SubscriptionWorker { + SubscriptionWorker { restore_observation_token: None, worker_id: worker_id(), runtime_id: Some("runtime-1".to_string()), @@ -660,8 +714,12 @@ mod tests { .unwrap(), }, ], - }; + } + } + #[test] + fn external_workdir_attachment_does_not_remove_workspace_worker() { + let mut worker = external_worker(); assert!(project_repository_keys_with(&mut worker, |_| { panic!("External Workdir must not require Repository projection") })); @@ -673,6 +731,110 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository() + { + use crate::store::{ + AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord, + WorkspaceRecord, + }; + let store = std::sync::Arc::new( + SqliteWorkspaceStore::in_memory_for_test() + .unwrap() + .with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()), + ); + store + .upsert_account(&AccountRecord { + account_id: "owner".into(), + kind: "user".into(), + handle: "owner".into(), + display_name: "Owner".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "workspace-1".into(), + owner_account_id: "owner".into(), + display_name: "Enrichment".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .await + .unwrap(); + let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1"); + store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: "workspace-1".into(), + worker: worker_ref.clone(), + display_name: "Durable name".into(), + profile: Some("builtin:coder".into()), + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2") + .unwrap(); + let committed = store + .worker_registry_projection("workspace-1", &worker_ref) + .unwrap() + .unwrap(); + // Change authority after capturing the payload, as can happen while async + // enrichment is queued. It must not re-query and replace the committed Worker. + let mut later = external_worker(); + later.state = protocol::subscription::SubscriptionWorkerState::Running; + store + .apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3") + .unwrap(); + let projected = project_registry_workers_async( + store.clone(), + "workspace-1".into(), + vec![committed.clone()], + ) + .await + .unwrap(); + assert_eq!(projected.len(), 1); + let worker = &projected[0]; + assert_eq!( + worker.state, + protocol::subscription::SubscriptionWorkerState::Idle + ); + assert_eq!( + worker.availability, + protocol::subscription::SubscriptionWorkerAvailability::Observed + ); + assert_eq!(worker.display_name.as_deref(), Some("Durable name")); + assert_eq!(worker.profile.as_deref(), Some("builtin:coder")); + assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1")); + assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1")); + assert_eq!( + worker.workdir_attachments, + external_worker().workdir_attachments + ); + let mut unresolved = committed; + unresolved + .observation + .as_mut() + .unwrap() + .worker + .workdir_attachments[0] + .repository_id = Some("missing-repository".into()); + assert!( + project_registry_workers_async(store, "workspace-1".into(), vec![unresolved]) + .await + .unwrap() + .is_empty() + ); + } + async fn run_with_events(events: mpsc::Receiver) -> SubscriptionResponse { let (control_outbound, mut control_receiver) = mpsc::channel(1); let (protocol_outbound, _protocol_receiver) = mpsc::channel(1); diff --git a/docs/development/backend-db-execution.md b/docs/development/backend-db-execution.md new file mode 100644 index 000000000..5eea19479 --- /dev/null +++ b/docs/development/backend-db-execution.md @@ -0,0 +1,159 @@ +# Backend bounded synchronous execution + +## Ownership and admission + +`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner. +Use `executor.run(move || synchronous_operation(...)).await` with owned input and +services. The closure owns the **whole** synchronous operation, including connection +acquisition, mutex wait, the existing SQL transaction, commit, and ordered local +publication. Do not split source authorization, receipts, CAS, audit, or saved +results around an await. Do not change connection ownership or introduce a read pool. + +Production stores share one DB worker and a queue of 32 jobs. Admission uses a +nonblocking queue send and a separate actual-job budget (worker count plus queue +capacity); rejection means the operation was **not accepted**. HTTP maps capacity +and executor unavailability to 503, not authentication failure or deletion conflict. +Tests can inject a dedicated executor with explicit limits, without process-global +settings or environment variables. Ordinary SQLite fixtures use `open_for_test`, +`in_memory_for_test`, or `from_connection_for_test`: their clones share their own +1-worker/32-queued owner, not the production singleton or another fixture's drain. +Server/Workspace fixture constructors likewise inject a lazy `HostExecutors` owner; +all child Workspaces and weak retention adapters use that same owner. Host and +cleanup still have the production 2-worker/16-queued limits. Dropping the final +sender closes its queue after accepted work finishes; idle fixture threads are +not kept in a process-global fixture cache. + +The restore intent journal and Workspace deletion use +`with_immediate_transaction`: read authority/journal state only after acquiring +SQLite write ownership. A DEFERRED read-to-write upgrade can return BUSY without +invoking SQLite's busy handler when another connection is writing. Moving these +whole helpers to an offloaded caller must not create an automatic mutation retry. + + +Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter: + +- does not interrupt, resubmit, or retry SQL; +- does not return its queue/job slot before execution and discarded-result cleanup; +- does not release the request lifecycle lease before actual completion. + +Synchronous post-commit continuations inherit the admitted lifecycle source when +submitting another bounded job. This is not implicit inheritance by detached Tokio +tasks. An operation panic has an unknown outcome even if SQL already committed; +return a structured unknown-outcome error and diagnostic, never retry the mutation. +A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all +jobs accepted before its barrier is first polled and waits for actual cleanup; it +is not a queue close and does not include later admissions. Lifecycle callers must +stop their producers/admission before destroying resources. + +## Runtime/network boundaries + +A synchronous Runtime workflow must not monopolize the sole DB worker while +waiting for callbacks that themselves need DB authority. The Server's separate +finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job +ownership mechanism, not an unbounded `spawn_blocking` queue. + +Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can +use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an +owned workflow needs async resource locks, `run_host_workflow` polls it on a Host +worker using the original Tokio handle; do not build a replacement runtime for +cached services and do not submit to and await the Host lane from inside itself. +Control/session guards needed by an accepted mutation belong to that actual job. +Subjektiv session Read/Search, subject response projection, general memory staging +and legacy consolidation-scope helpers also use Host ownership: they can +synchronously call Runtime/network services outside their SQL transactions. +Pure DB-resident Subjektiv scope and memory projection helpers stay on the DB lane. +Host admission must not be acquired again from an already-owned Host workflow, +including indirectly through a Runtime's mandatory Backend callback. + +Mandatory Session attribution has Subject-only authority and whole attribution +SQL on the independent DB lane. Store-only scope discovery distinguishes Subject, +current Job consolidation, and legacy consolidation authority; attribution rejects +both consolidation roles **before** any Runtime profile/tag lookup. Legacy +consolidation checks for mixed callers remain on Host. Subject-only memory +operations (resident context/summary, proposal validation, receipt lookup, explicit +staging) use the same store-only authority boundary and DB ownership. Runtime +bootstrap/restore can request resident context while its Host parent is waiting. +This is not permission to bypass Workspace deletion admission: only the already +allowed exact callback endpoints may reenter deletion drain. + +Backend Job cleanup has a third finite lane (two workers, 16 queued jobs), so a +long Run teardown wait cannot starve ordinary Host control/stop requests. It is +admitted before its durable cleanup claim. Rejection leaves Pending/Failed +ownership recoverable instead of abandoning an Executing claim. Ticket +checker dispatch and timeout processing use bounded admission; overload leaves the +saved reservation/deadline diagnosable. They are not automatically retried after an +unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof +of model processing. + +Remote subscription and Workdir authorization are async callbacks: their complete +live binding check and fresh request signing are offloaded together. Cold protocol +source lookup in `connect_workspace_worker_protocol` is also awaited as an owned +DB stage, rather than invoking its synchronous signing callback on Tokio. +`WorkspaceWorkerRuntime::observation_source` is fallible: live binding/signing +failure and bounded admission errors propagate explicitly, not as a missing +source or a fallback to a configured bearer token. Synchronous CLI/migration +callers and existing synchronous Runtime adapters retain their explicit blocking +boundary. + +## Caller recipe (including assignment APIs) + +```rust +let store = api.store.clone(); +let workspace_id = api.workspace_id().to_owned(); +let worker = worker.clone(); +let record = api.store.db_executor().run(move || { + store.get_worker_registry(&workspace_id, &worker) +}).await?; +``` + +This is an illustration, not a new authority API. Assignment delivery scan, +reserve, prepare, acknowledgement and start authorization must keep their existing +whole transaction helpers. Additional assignment diagnostics, answer mutations, +and Ticket queries use the same owned caller boundary; do not add a second ledger +or reconstruct processing evidence from Worker liveness. + +Worker discovery uses the same whole `workers_response` DB stage as Worker list +and resource detail: projection snapshot, Workdir registry/links, and authority +projection never acquire the shared connection on Tokio. Source-proof requirement +and pure query/cursor validation still precede admission; overload remains 503. +Runtime resource list/detail use an accepted Host workflow for synchronous Runtime +summary I/O and await independent DB binding/audit reads. Launch options are also +mixed Runtime/config helpers and use Host, not the sole DB worker. Runtime +registration validates its signing identity in a whole DB stage, scans existing +Workers on Host, then owns SQL key/CAS mutation, local registration publication +and response observation in one accepted workflow. Metadata updates likewise keep +commit and publication together after waiter cancellation. Trust-key revoke also +owns whole CAS/audit SQL, expectation removal, broker unregister/Shutdown and +response observation in one accepted Host workflow. Future request proof denial +alone does not terminate existing subscriptions. Host admission must precede the +mutation, so overload cannot commit a revoke then fail on response admission. +Public resource/detail helpers admit Host ownership; already-owned workflows call their `_on_host` +variants directly, never await another admission to the same occupied lane. +Connection tests ping on finite Host ownership and then await the whole current +binding fence on DB. A replaced/revoked binding cannot attest the earlier ping. +Do not confuse a Runtime summary callback with a pure stored Worker projection +or run network waits on the sole DB worker. + +## Regression evidence + +- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread + timer/control task progresses; accepted/cancelled queue bounds; rollback; panic + before/after commit; discarded reply cleanup and retained authority; child jobs. +- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope + opening/mutation under a separate SQLite connection's write lock; overload; + cancelled HTTP jobs and Workspace deletion drain; same-budget fixture isolation, + saturated/cancelled Host jobs versus cleanup and paused Tokio time; restore intent + contention before the first authority read, with one persisted tuple and no retry. + The terminal checker fixture keeps its original Tokio driver alive through accepted cleanup, + then shuts down projection producers and drains actual DB jobs. +- `worker_projection::tests`: real SQL contention, commit/payload publication, + ordered snapshot registration, and replacement draining a cancelled consumer's + accepted SQL before applying a new Runtime snapshot. +- Auth/proof and Runtime subscription tests protect availability classification, + replay rejection, and no bearer fallback. +- `workdir`'s `http-client` tests protect awaited fresh authorization and exact + request claims before any HTTP send. + +These fixtures use temporary authority data and observable synchronization, not +live databases or a fixed sleep to guess completion. This change does not claim a +latency benchmark or a measured production incident. diff --git a/docs/report/2026-10-12-t748-caller-inventory.md b/docs/report/2026-10-12-t748-caller-inventory.md new file mode 100644 index 000000000..3fe21efba --- /dev/null +++ b/docs/report/2026-10-12-t748-caller-inventory.md @@ -0,0 +1,287 @@ +# T-748: async caller ownership inventory and scoped repair evidence + +## Current status and evidence boundary + +MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq13 requested changes for published +`ebce7ab7b1230654b8eb5340afca8de835f24f12`. The independent review confirmed the +revoke cancellation/publication fix, but found three further direct async callers. +Earlier full suites exercised successful outcomes without proving every composed +caller boundary. Repeatedly treating a helper's apparent purpose as its lane +classification missed Runtime lookup in Ticket projection and post-await reads. +A successful suite is not a substitute for the entry-to-publication inventory. + +The seq13 and adjacent fixes were committed as `5873f250`; embedded protocol +and Feature shutdown fixes as `4d981194`. The remaining confirmed caller-placement +and Worker stop repairs are included in the follow-up below. The read-only audit +children were released after result-observation gaps; that release was not treated +as completion. No audit-to-implementation approval gate is imposed. +No target integration, Ticket conclusion, live update or attachment cleanup is +performed. T743 domain APIs are not copied or implemented here; their owner still +needs normal composition and exact combined-source review. + +The tables and line numbers below preserve the **pre-repair audit snapshot**. +“Unfixed”, “pending”, and “remaining” in those historical sections describe that +snapshot, not the current implementation. The final section supersedes their +placement/stop status and explicitly distinguishes tested boundaries from limits. + +Audited snapshot `server.rs` SHA-256: +`dae05a2649f5d4f884e8118ba4ce6baa902de6e61727c20e97c0afa3661ef3a6`. +Audited snapshot `worker_operations.rs` SHA-256: +`e3ad2327370440fdc40fb2c76f9604b984667e3472570d76145f4d0987d343fd`. +These file hashes identify the inspected implementation, not a provider review ref. + +## Current confirmed ownership changes + +| Entry / composed family | Authority and synchronous dependencies | Current owner / completion boundary | Evidence and limits | +|---|---|---|---| +| `ticket_implementation_cancel` → `scoped_cancel_ticket_implementation` | Ticket authority/assignment reads; Runtime cancel; current-session close; intact assignment cancellation transaction; detail | One accepted finite Host workflow owns cancellation through transaction and response. No Runtime wait on DB, no nested Host admission | Pre-authenticated generated POST route; real separate SQLite writer holds actual cancellation transaction; timer/control progress; cancelled waiter retains actual lease, then one operation/cleared assignment after drain. Saturated Host returns 503 without mutation. Existing live Coder cancellation policy passes | +| `merge_request_selector_repair` → `scoped_repair_merge_request_selector` | Browser auth/explicit confirmation; MR store; current assignment; provider ref; intact repair/audit; public projection | Auth remains an owned DB read; confirmed post-auth mixed stage is one finite Host job | Browser auth is drained before real SQL/shared-mutex contention; post-auth stage stays Pending without blocking Tokio and retains cancelled Host lease. Imported NULL-selector/no-assignment fixture rejects without repair/thread effects. Host overload preserves confirmation priority. Successful provider repair is not claimed by this narrow fixture | +| restore routes → `restore_runtime_worker_with_context` → `project_worker_restore_response_owned` | Existing authorized Host restore returns historical result; then whole stored Worker/links/Workdir/capability projection | Whole independent DB stage. A result with no Worker remains a pure response and needs no DB admission | Real SQL writer/shared-mutex hold; timer and independent Host control progress; bounded overload; cancellation retains actual DB lease; historical state preserved and no registry observation write. This tests post-restore policy, not a complete restore endpoint/E2E | +| Ticket detail/show/query and alias `get_ticket`; edit/event detail and assignment preparation | `TicketAuthority` projection includes open-MR `RuntimeTicketSourceCommitResolver` → synchronous provider HTTP | Mixed stages use finite Host, not sole DB. Subsequent pure assignment mutation stages keep their existing owner | Detail/query held loopback `/v1/repository-refs/observe`; unrelated DB/timer/control progress; cancel retains actual Host budget; overload stays 503. Provider-placement fixture is explicitly unsigned, not proof of signed request authority | +| Ticket tracking/case decision authority | Retained assignment/CAS/body/event checks plus `find_workspace_orchestrator` → live `RuntimeRegistry::worker` | Whole existing decision helper on finite Host; transaction and notification order unchanged | Held loopback live Orchestrator lookup for both tracking/case; independent SQL/timer/control; cancelled actual Host lease retained; failed live authority leaves Ticket unchanged. Existing non-assignee/browser bypass denial passes | +| Browser Ready | Existing mutation/notifications, followed by detail (including source ref observation) | Mutation and detail use one accepted Host job and explicit `execute_ticket_rest_operation_on_host`, never a second Host admission | Fake promptly denies notification GET, holds the subsequent source-ref POST; exactly one Ready event/readiness is durable before held detail; SQL/timer/control progress and cancelled Host ownership persist. Rejected fixture setup/misidentified notification GET are excluded | +| Worker control input/cancel/stop/restore source preflight | Synchronous live source Runtime lookup precedes `resolve_async` and operation admission | `authenticate_worker_mutation_source_owned` admits finite Host; already-owned synchronous callers retain the original helper | Actual control handler's held source GET proves DB/timer/control progress and cancelled finite Host budget; saturation returns availability, not source denial. The fake's lookup fails before an actual Worker cancellation; no product process is used | +| Worker observation capture | Source lookup, initial SQL grant, control lock, live grant re-fence and target Runtime lookup, awaited cold protocol connection | Source on Host; initial grant on DB; actual re-fence/live lookup on Host owns the acquired control guard; cold protocol signing retains existing DB stage | Source/permission/cold-signing contracts remain covered separately. Full capture success/held-protocol composition is not claimed by the new source lookup fixture | + +## Regressions and causal checks + +New `server_residual_execution_tests.rs` contains eight focused tests, including +three read/Ready and two decision-authority table cases. Exact old methods or the +old direct stored-projection placement were restored diagnostically in turn for +cancel, repair, restore, detail, tracking, and control. All six fail at their expected +first-poll/shared-mutex/independent-SQL boundary. Five-second fixture watchdog +release and its induced SQLite BUSY/panic are diagnostic bailout, not progress. +The product source is restored in `finally` after each case. + +Artifacts (local, not tracked product code): +- `target/t748-residual-old-{cancel,repair,restore,detail,tracking,control}-red.log` +- `target/t748-residual-final-restored-green.log`: **26 DB-boundary tests pass** +- `target/t748-residual-decision-policy.log`: non-assignee/browser bypass policy passes +- `target/t748-residual-root-check.log`: root compile closure passes +- `target/t748-residual-provisional-full-suite.log`: original default-parallel Server + **979 passed / 0 failed / 1 existing ignored**, binary11; **178.38 seconds**. + Only the three deliberate executor unknown-outcome panic diagnostics appeared. + No retained test/repro processes were observed after completion. +- `target/t748-residual-provisional-workdir-{default,http}.log`: default153 and + http-client156 tests pass. This does not complete the pending transitive audit. +- fmt/diff and state-contract names **558 / 0 unapproved / 0 stale** pass. + +All Cargo validation uses `target/t748-candidate-build`; no parallelism reduction, +production budget increase, skipped tests or mutation retry was introduced. +Initial test-only private enum/state/DTO mistakes failed compilation; missing +repository seeding and Ready notification GET confusion were diagnosed and fixed. +Those attempts are not semantic proof. The full-suite result above is provisional: +it does **not** conclude the caller inventory or approval. + +## Remaining inventory (not complete) + +`target/t748-async-entry-candidates.tsv` records 661 lexical async-entry candidates +from Server, Worker operations, auth, projection and subscription sources. Visible +lane markers/await counts are discovery hints, not an AST/type-checked call graph +or proof that all transitive operations are safe. Forwarders, actual handlers, +service adapters and callback/background callers must be matched by name and +traced through their real dependencies before marking a family inspected. + +| Inventory work | Current status / next evidence | +|---|---| +| Ticket/MR/Objective/Memory/Subjektiv/Backend Job: canonical/alias/browser/Runtime entries, assignment/routing/decision/projection/publication and callback edges | First committed child result inspected. Major handler owner table returned, but follow-up final result was not observable. Parent released the child and is directly checking remaining adapters/forwarders/launch edges. Macro trait dispatch and representative Objective/Memory/Ticket forwarding plus legacy Memory/cleanup owners inspected; complete per-operation mapping remains open | +| Auth/Runtime/Worker/control/observation/resources/Workdir/repository/Drive/settings/config/catalog/uploads/sessions/subscriptions and lifecycle background edges | First committed child result inspected; parent verified the residual structures below against actual source. The follow-up result was not observable after two Idle notifications; parent released the read-only child and directly traced the remaining four edge groups below. This is not delegated completion evidence | +| Existing pure DB, mixed Host, mandatory Subject callback and accepted publication boundaries | Earlier documented evidence remains relevant; unchanged code does not exempt newly found composed callers from tracing | +| `WorkspaceWorker::stop` first Host result → waiter-owned session close → second Host stage | Parent verified the split in actual source. Structural ownership gap, not yet a cancellation reproducer; tracked below rather than silently marked safe | + +## Additional residuals found before republication (unfixed) + +The first two read-only audit results were inspected through their committed +sessions. The parent then read the concrete handler/helper source listed below. +These are **static execution-placement/continuation findings**, not new successful +endpoint, cancellation or overload tests. None has been repaired or published in +this snapshot. The provisional suite above therefore cannot establish safety for +these families. Lower recovery/transport edges are still being traced by the +parent after releasing the read-only children; this is not independent review. + +`S` below means `crates/workspace-server/src/server.rs` at the frozen hash; +`O` means `crates/workspace-server/src/server/worker_operations.rs`. + +| Entry / family | Entry → authorization → synchronous work → completion | Verified residual and required boundary | +|---|---|---| +| `repository_access_projection`, `repository_ssh_credential_{list,create,generate,get,delete,public_key,rotate}`, `repository_ssh_host_trust_{list,put,get,delete}` | ServerApi entries `S:7879–8107` directly await handlers `S:12934–13129`; owner authentication is awaited first, then active config projection / secret / trust / CAS / audit helpers execute synchronously | Auth ownership does not carry past await. Parent read all handlers and actual `repository_access.rs:391–436` connection reads. Whole existing synchronous operation needs an owner; pure DB and any external provider work must be classified by dependency, not by the SSH label | +| `repository_create` → `scoped_create_repository` | Owner authentication await → validation → `config_store.insert_repository` → intent/replay classification → response, `S:23157–23214` | Insert is outside an accepted job. Preserve the existing intact insert/intent/commit helper and successful replay vs conflicting intent semantics | +| `repository_ssh_connection_probe`, `repository_ssh_host_trust_confirm` | Owner and active binding awaits → catalog / `probe_repository_ssh_connection` → Runtime observation → trust read/CAS/write → response, `S:23247–23358` | Post-auth mixed work has no finite Host owner. Confirm must keep observed host-key selection and fingerprint comparison before the existing trust mutation | +| `worker_session`, `worker_session_history`, `scoped_worker_session_attachment` | Workspace checks → synchronous Runtime Session API → response/body, `S:8696–8769,12011–12069`; registry facade `hosts.rs:2129–2195` | No finite Host admission at these request boundaries. Binding/signing and remote transport dependencies must stay off Tokio and out of the sole DB lane; preserve Session diagnostics and response limits | +| `runtime_worker_attachment_upload` → `upload_runtime_worker_file_contract` | Grant claim → registry resolution → Runtime upload → grant completion or cancellation artifact deletion, `S:28081–28152` | Whole body-upload operation is still direct synchronous mixed work, distinct from the already-owned grant/cancel/delete routes. Admission must precede grant claim; accepted upload must own completion and cancellation cleanup | +| Drive Worker authorization / `drive_grant_create` | `server_drive_grants.rs:54–74,126–166`: source/header checks → `current_worker_identity` or live Runtime worker → later DB grant precheck/mutation | Pre-DB source/live Runtime lookup has no Host owner. `S:18244–18292` confirms `current_worker_identity` directly reaches synchronous `runtime.worker`. Retain no-actor-fallback policy; owner unavailability must not be misreported as authority denial | +| `current_worker_workspace_config_{get,attach,observe,read,commit}` | `S:11625–11699` directly invokes `workspace_config::identity`; `server_workspace_config.rs:65–74` reaches the same synchronous Runtime source helper before later owned DB stages | All five logical-config entries share the unowned identity preflight. Later accepted persistence/publication does not cover this earlier stage | +| `workspace_workdir_list` → `scoped_list_working_directories` | DB job `S:25943–25959` → summaries `32974–32999` → all Runtime workdir observations `33515–33524` → list/detail Runtime calls and SQL reconciliation `33421–33467` | Mixed Runtime/SQL helper is incorrectly on the sole DB worker. Preserve live reconciliation and diagnostics, but move the whole mixed stage to finite Host rather than pretending the list is stored-only | +| `workspace_config_tree_commit` → `scoped_commit_workspace_config_tree` | DB job `S:13444–13460` → prepare → repository-access projection → config commit → Runtime prompt publication, `13498–13517`; remote publication `hosts.rs:5090–5099` | SQL commit is followed by synchronous network publication inside sole DB job. Keep commit-before-best-effort-publication and accepted ownership; do not leave the continuation on the waiter. Logical config's separate owned Host dispatch is not this UI path | +| `repository_{list,detail,log}` and aliases | DB jobs `S:29108–29183` → live repository reader → summary/log provider; `repositories.rs:276–375` invokes Git subprocess for materialized local repositories | Provider work is not pure DB. Mixed catalog/provider work needs finite Host, without changing existing diagnostics for unmaterialized sources | +| `worker_control_stop`, `runtime_worker_stop`, `runtime_worker_stop_alias` → `WorkspaceWorker::stop` | `O:379–425`: grant/lock/re-fence → Host #1 Runtime stop returns result/control guard → waiter session lock/close → Host #2 linked Workdir observation/SQL | Structural ownership gap confirmed by parent: accepted Runtime stop has no owner for session close and subsequent Workdir synchronization if its waiter vanishes. `S:18609–18619` awaits real session close; the second Host admission also occurs after side effects. Whole accepted stop must own this continuation without nested Host acquisition. Cancellation reproduction remains pending | + +### Parent takeover of the four remaining Runtime edge groups + +The Runtime child's first result remains useful, but its follow-up final result +was not available through authorized observation. The parent did not infer +completion from Idle and did not loop Submit or inspect raw session storage. +See `docs/report/2026-10-12-worker-observation-followup-gap.md` for the bounded +observations and handling. The parent read the following actual source directly: + +| Edge | Source and actual stages | Classification / unresolved contract | +|---|---|---| +| Embedded protocol connect and method pump | `S:31845–31902` directly calls synchronous `attach_worker_protocol_scoped` before spawning its async pump; the pump directly calls `send_connected_protocol_method_scoped`. `worker-runtime/src/runtime.rs:3486–3598` takes synchronous operation/state locks; Shutdown performs `begin_stop_operation` and `commit_stopped_worker` (`4359–4445`), with durable record/snapshot writes in fs-store builds (`5256–5269`) | **Additional confirmed placement residual**: Backend async connect/pump is not an accepted finite Host owner. The existing endpoint identity/settled-operation checks and commit/publication helpers must remain intact; no Runtime-store refactor is needed | +| Workdir creation reservation → Runtime result → finalization | `S:26316–26609`: DB preparation/reservation, Host runtime selection/observation/create, then waiter-admitted failure or final registry/receipt DB jobs. Same-id reentry observes the reserved Runtime Workdir before attempting create (`26473–26517`) | **Confirmed continuation split**, with durable same-operation recovery intent, not automatic mutation retry. Existing recovery does not make the original create job own finalization after its waiter disappears. Final classification/regression must cover accepted Runtime outcome, final receipt, cancellation and overload without creating a second provider operation | +| External grant fence → attachment/provider cleanup | `S:25595–25709`: DB authority/read or revoke/fence/audit, then waiter-owned cleanup/provider stop; terminal GET/revoke explicitly retries cleanup. Cleanup itself commits pending, awaits release, then admits completion/audit DB (`25719–25799`). Expiry task retains its async owner and retries (`24790–24854`) | **Confirmed continuation split plus explicit recovery**, not a claim that revocation failed or is unsafe after its DB fence. Grants remain denied by their committed fence. Need causal cancellation/lease evidence for cleanup and preserved retry/expiry policy before choosing the smallest ownership fix | +| Command provider acceptance → registration/audit | `S:19507–19631`: Host identity, DB attachment fence, async session/provider, then synchronous command-handle registration. External `operate` admits audit-requested DB, sends provider request with explicit cancel guard, receives result, then admits audit-completed DB (`888–1024`); External command-handle registration follows another await (`1372–1428`). Local command start retains Runtime resources in its actual task and owns terminal-close admission (`workdir/src/local.rs:1831–1897`) | **Partially protected / continuation question**. SQL jobs are owned and local actual command resources are retained; explicit external cancel is not SQL retry. Provider acknowledgement-to-registration can cross a cancellable audit await. Do not assert an orphan process or whole command success without a focused protocol/registration reproducer; do not rewrite external cancellation semantics speculatively | +| Catalog and single-workspace shutdown | `S:12222–12279`: catalog waits for HTTP serve, drains Worker projection and owns Feature shutdown DB (`2651–2664`); single-workspace `serve` aborts its hook but calls synchronous Feature shutdown directly. `feature-storage/src/lib.rs:220–241,521–523` fences and waits on connection close. Executor `drain` (`db_executor.rs:290–315`) covers only jobs accepted before its first poll, not later submissions | **Additional confirmed direct shutdown placement residual** in single-workspace serve. Separately, catalog does not demonstrate hook/expiry/Host/cleanup producer quiescence before storage shutdown. Whole producer/drain ordering remains a lifecycle question; no live shutdown was invoked to test it | + +Domain generated adapters, remaining Tool/WIP forwarding, Backend Job / legacy +Memory launch lower edges remain under follow-up inspection. Existing domain SQL +helpers are not being reimplemented or globally refactored; ownership tracing is +separate from re-auditing every SQL semantic branch. None of the additional rows +above has new executable regression evidence yet. + + +T743's additional assignment page/diagnostics/answer APIs are absent from this +source. Ticket seq8–9 identifies their separate owner's normal composition with +the published T748 facade. This inventory is **not** an attestation of that future +combined source; no duplicate API or combined approval is introduced here. + +## Republication conditions + +Review-ready requires a consolidated caller-name list with each entry's auth, +SQL/provider/Runtime stage, post-await projection, post-commit publication and +owner identified; confirmed residuals repaired with focused causal regressions; +remaining uncertainty explicit; and final root/changed-crate/affected-feature, +format/diff/name checks on that final source. Only then normal same-selector push, +provider exact-HEAD verification and fresh trusted independent review of this MR. +No new Reviewer Runtime or separate Coder is created. Read-only audit children do +not grant review verdict or integration authority. + +## Scoped caller repair and validation history + +The same Ticket branch is retained for advancing the existing MR. This section +records validation history, not approval or integration. The final-validation +section below supersedes the intervening failed runs without deleting their evidence. +The existing domain helpers, +authorization fences, receipt/replay rules, and transaction/publication order +remain intact. No automatic mutation retry, production limit increase, nested +Host admission, or test parallelism reduction was introduced. + +| Repaired family | Current owner / executable evidence | +|---|---| +| Repository access/SSH credential and host-trust CRUD, Repository registration | Whole post-owner-auth synchronous operation on DB. Registration regression observes the **config store's actual connection** waiting behind a separate SQLite writer, then cancels the waiter; after drain exactly one Repository exists and the same registration intent returns HTTP200/replayed without another row | +| SSH probe/host-trust confirm | Whole post-owner/binding-auth mixed probe/observed-key selection/fingerprint comparison/trust mutation on Host. Existing narrow SSH/trust policy tests pass in the full Server suite; no successful real SSH connection or host-key exchange is claimed | +| Repository list/detail/log and aliases | Live Git/catalog helpers on Host, not DB. With DB admission saturated, empty catalog and absent-key detail/log retain their ordinary success/404 responses. This regression proves lane selection, not held Git subprocess behavior | +| Worker Session/history/attachment | Typed Session service and attachment handler admit Host before synchronous Runtime observation. Loopback held Runtime lookup proves independent SQL/timer/control progress, cancelled actual lease retention, bounded overload and release after actual drain | +| Body upload | Host admission precedes the upload grant claim and owns resolution/upload/completion/cancellation cleanup. The same loopback table holds the Runtime lookup after claim; cancelled waiter leaves actual owner, and provider refusal completes the claim back to Pending. Successful artifact upload/delete and established remote sessions are not exercised by this refusal fixture | +| Drive Worker authorization / grant create; all five logical-config identity entries | Live source/target preflight uses Host; later DB authorization and mutations retain their previous transaction owner. Three held-lookup table cases prove independent SQL/timer/control, cancelled lease retention, and typed HTTP503 availability (not false authority denial) when Host admission is full. Existing no-actor-fallback and source-fence policies remain in the suite | +| Workdir list | Whole live Runtime reconciliation/SQL helper on Host. Held Runtime list lookup in the Session/Workdir/upload table proves sole DB remains free and cancellation retains the actual Host budget | +| UI config commit | Whole existing evaluate/project/commit/best-effort prompt publication helper on Host. Loopback held prompt POST confirms the changed tree is already durable while independent DB/timer/control proceeds. Cancelling the waiter retains publication ownership; refused publication neither repeats nor rolls back the commit | +| Worker stop canonical/alias/control entries | One accepted Host workflow owns Runtime stop, session fence/close, stored read and linked Workdir synchronization. An in-process Runtime emits its stop receipt while a real session fence is held; after waiter abort and fence release the registered session is removed. Restoring the exact old stop method makes this test fail at the missing cleanup assertion; restoring the fix passes. No real Worker/product process is spawned or stopped. Linked Workdir provider success is not claimed by this session-only fixture | +| Embedded protocol connect/method; single-workspace Feature shutdown | `4d981194` owns these operations on the existing finite Host/DB lanes. Focused protocol and actual Feature connection shutdown regressions remain green in the final suite | + +Final logs in the Workdir (build artifacts, not committed product content): +- `target/t748-stop-old-placement-red.log`: old stop fails **missing cleanup**, not timeout. +- `target/t748-stop-restored-green.log`: restored accepted-stop ownership passes. +- `target/t748-caller-host-green.log`: Session/history/attachment/Workdir/upload table passes. +- `target/t748-repository-create-green.log`: actual SQLite commit/cancel/replay passes. +- `target/t748-drive-config-preflight-green.log`: all three live authority cases pass. +- `target/t748-ui-prompt-owner-green.log`: commit-before-publication passes. +- `target/t748-repository-provider-green.log`: all three Repository read lanes pass. +- `target/t748-caller-final-root-check.log`: workspace-root `cargo check` passes. +- `target/t748-caller-final-server-suite.log`: original default-parallel Server + **988 passed, 0 failed, 1 existing ignored**, plus **11 binary tests**, 185.38s. +- `target/t748-caller-final-workdir-default.log`: **153** tests pass across targets. +- `target/t748-caller-final-workdir-http.log`: **156** tests pass across targets. +- fmt/diff checks pass; `target/t748-caller-final-names.log`: **558 / 0 unapproved / 0 stale**. + +Test setup errors are excluded from semantic evidence: the first stop fixture +omitted the Runtime's live Worker summary; the Repository busy observer initially +watched the general store rather than the config store's distinct connection. +Both were corrected before the successful runs. Missing package/Python command +and compile-time DTO/Duration/Drive-error mistakes also prove no product behavior. + +### Explicit limits, not an expanding audit gate + +The four lower-edge rows above separate confirmed direct placement repairs from +Workdir create / external grant cleanup / provider acknowledgement continuation +questions and catalog producer quiescence. The latter are not redesigned here, +per the user's instruction not to expand speculative Workdir/grant or unrelated +Feature work indefinitely. Their existing durable recovery, explicit cleanup +retry and provider cancellation policy are preserved, not reclassified as +end-to-end cancellation proof. This report is not a mechanically complete +transitive call graph or an assertion that every future/composed caller is safe. +T743's separate new API composition still requires its owner's exact combined +source validation/review. None of these limits constitutes Ticket conclusion, +MR approval, target integration or permission to update the running environment. + +### Post-suite focused validation blocker + +After the successful full suite above, the current candidate was checked with +`cargo test -p yoi-workspace-server --lib db_execution_tests` using the same +candidate target directory and ordinary parallel execution. Result: **31 passed / +1 failed**, in 8.06s. Failure: +`cancelled_runtime_key_revoke_commits_audit_and_removes_broker_registration` +reported the shared `observed` helper's two-second `Elapsed(())` timeout, without +identifying which of its several lifecycle waits timed out. Source inspection +confirms projection shutdown is already present in this fixture; it is not +legitimate to attribute the failure to omitted projection shutdown, treat it as +load-only, increase limits, reduce parallelism, or rerun until green. + +Evidence is preserved in `target/t748-caller-final-boundaries.log`. The prior +full-suite success is real but insufficient to declare final validation complete. +The new caller and stop cases all passed in this focused run. Next diagnosis must +identify the exact wait and inspect actual HTTP/job/broker ownership events before +choosing a fixture or product repair. No normal push, fresh review, approval, +integration or Ticket conclusion was performed for this candidate. + +## Continued diagnosis and final validation + +Returning the failed test as a stop condition was a workflow mistake, not an +external outage, lost authority, or additional audit approval gate. Work continued +on the same branch. The failed run above is retained as evidence. + +The `observed` helper now reports its caller location. A diagnostic 32-case run +passed; that result alone was **not** treated as a diagnosis or used to erase the +failure. Source tracing identified an uncontrolled cold broker signing producer +in the revoke fixture. The test now receives the actual signed WebSocket +handshake and drains its completed DB work **before** holding the revoke writer. +The handshake remains pending, so broker shutdown is still tested. An early HTTP +response is selected against actual SQLite entry, reporting status/body rather +than waiting for an event that can no longer occur. The original failed run did +not identify its exact wait; attributing it conclusively to signing would exceed +the captured evidence. + +Subsequent full runs exposed the Ready table's distinct failure **before** source +POST arrival (caller location, not timer/control failure). Artifacts: +`target/t748-post-fixture-server-suite.log` and +`target/t748-final-framing-server-suite.log`. The HTTP fake previously assumed +one TCP read contained the request line and answered without consuming a body. +It now parses the complete header/body; a deterministic two-byte-fragment reader +proves complete frame consumption and truncated-body rejection. This is a real +fixture flaw but does not by itself establish the historical timeout cause. + +Ready setup includes mutation and best-effort notification before the source +POST. The held-provider arrival now uses the existing five-second fixture +watchdog as its setup bound, selected against handler completion. The subsequent +**timer and independent SQL/control observations retain their two-second budget**; +actual held-request cancellation/lease/overload/cleanup assertions are unchanged. +Production admission limits, normal test parallelism, and provider timeout are +unchanged. This setup-bound change is explicit, not a hidden limit increase or +claim that Ready completes within two seconds under a whole-suite workload. +Restoring the exact published old Ready method still fails this revised test +(`target/t748-ready-old-boundary-red.log`, Ready returns only after synchronous +provider wait); restored code passes, so the offload distinction is not weakened. + +Final code validation after these fixture changes: +- `target/t748-boundary-setup-fixed-green.log`: **33 DB-boundary cases pass**. +- `target/t748-final-setup-root-check.log`: root compile closure passes. +- `target/t748-final-setup-server-suite.log`: original default-parallel Server + **989 passed / 0 failed / 1 existing ignored**, **11 binary tests**, 189.66s. +- `target/t748-final-setup-workdir-default.log`: **153 pass** across targets. +- `target/t748-final-setup-workdir-http.log`: **156 pass** across targets. +- `cargo fmt --all -- --check`, `git diff --check HEAD`, and state-contract names + **558 / 0 unapproved / 0 stale** pass. + +No failed semantic run was merely repeated until green: the intervening source +changes and the limits of each diagnosis are recorded above. These results support +publication for fresh exact-source independent review, not reviewer approval, +MR integration, Ticket conclusion, or updates to the live dogfooding environment. diff --git a/docs/report/2026-10-12-t748-suite-diagnosis.md b/docs/report/2026-10-12-t748-suite-diagnosis.md new file mode 100644 index 000000000..ed594bde6 --- /dev/null +++ b/docs/report/2026-10-12-t748-suite-diagnosis.md @@ -0,0 +1,444 @@ +# T-748: default-parallel suite failure diagnosis + +## Sources and preserved observations + +- Original local candidate: `61533fadb6b39465af60cb55aed2b4d0650319cd`. +- Comparison source: `56795fb92cb9def9bcd701bd51d92e99526f8eff` (develop/T742). +- Same implementation branch: `work/T-748-bounded-db-execution`. +- The original `target/t748-server-tests.log` remains untouched. Its 97 FAILED + lines and 600-second timeout were symptoms, not proof of one cause. +- Initial diagnosis used `cargo test -p yoi-workspace-server -- --nocapture`, + default libtest parallelism, without changing queue limits, features, or startup + settings. `target/t748-diagnostic-environment.txt` records the environment; + this host reports 24 available CPUs. Diagnostic captures were bounded to + 90/150 seconds to obtain errors before the original timeout, not counted as + successful full-suite runs. + +Logs under `target/` are local diagnostic artifacts, not tracked review inputs. +This report records their meaning and the permanent regressions that reproduce it. + +## 1. Different fixtures consumed one global Host admission budget + +The first non-intentional errors in `target/t748-server-nocapture.log` were +`DbCapacityExceeded` in manual Worker setup (`server.rs` at the then-current +line 50614), followed by assignment fixture construction (52990/53045) and +HTTP 503 responses. The deliberate before/after-commit executor panic tests +are different: those passed and verified unknown-outcome diagnostics. + +Temporary submit/start/cleanup instrumentation recorded submitting test names, +closure types, actual job IDs and the execution-owner identity. It showed a +single Host lane with `max=18`, with accepted jobs from unrelated fixtures. +A rejection snapshot contained `{32,45,46,...,61}`. The complete diagnostic +capture later contained cleanup for every submitted Host job on that owner; +it did not show permanently abandoned accepted Host jobs. Instrumentation was +removed before the source was committed/published. + +`target/t748-live-process-18.txt` sampled the test process and threads at the +actual rejection. That sample had 126 threads; the Host workers were runnable, +and the process was using approximately 17 CPU cores. There was no captured +thread-creation EAGAIN/unavailable error. This evidence does not establish an +OS thread/CPU hard limit as the cause. No test processes remained after the +bounded diagnostic run or the completed suite. + +A representative failure passed alone (0.56 seconds); the eight logging cases +passed with four test threads (2.46 seconds). The isolated comparison source's +full default-parallel suite passed: 931 library tests, one pre-existing ignored +migration fixture, 11 binary tests; library duration 168.74 seconds. This did +not prove that all new failures were capacity-related, but did distinguish the +new shared-admission rejection from an observed baseline failure. + +### Ownership fix (`f3d4aba5`) + +Production budgets remain shared and finite: DB 1 worker/32 queued, Host 2/16, +cleanup 2/16. Ordinary fixtures now explicitly own a finite DB executor and lazy +Host/cleanup owners at those same limits. API clones, child Workspace APIs, and +weak retention adapters use their injected owner. No global fixture lane cache, +parallelism reduction, unlimited queue, or capacity-wait retry was added. + +The permanent saturated-Host regression fills both running slots and all 16 +queued slots, cancels waiters, advances paused Tokio time, and still observes +rejection until actual completion. Another fixture and the separate cleanup lane +can progress; draining restores the original budget and completes exactly 18 +accepted jobs. Existing real-SQLite cancellation/lease/drain tests remain active. + +## 2. SQL offload invalidated fixture assumptions about first poll and publication + +After Host isolation the default-parallel suite finished in 171.57 seconds: +954 passed, three failed, one existing ignored. Actual errors were: + +- removal subscriber saw a legitimate queued Runtime stop Upsert before Removed; +- external session-open expiry exceeded its one-second bound while its DB lane + was still shared across other fixtures; +- grant replacement returned success because one initial poll had only submitted + the offloaded authority read, not reached the held original-grant lock. + +The ordinary DB fixtures were isolated too. The one-second expiry assertion was +not increased. Grant testing now drains its actual authority read then polls the +saved response to reach the held lock before replacing the grant. Removal testing +observes the committed stopped projection before taking the removal baseline. +The Removed payload assertion is unchanged. These are observable synchronization, +not sleeps or retries. Focused expiry, grant, and removal tests all passed. + +A captured cleanup job also attempted to poll a Tokio timer after its fixture +runtime had shut down. The terminal checker fixture now retains that original +runtime through accepted Host and cleanup jobs, then stops projection producers +and drains actual DB work before ending. Terminal SQL state is not treated as +actual cleanup completion. The subsequent full run contained only the three +intentional executor panic diagnostics, not the shutdown-context panic. + +The synchronization/teardown changes are in `df9d056d`. + +## 3. Restore intent's DEFERRED read-to-write upgrade returned SQLite BUSY + +The next default-parallel run finished in 171.48 seconds with 956 passed, two +failed, one existing ignored. Both failures were SQLite `DatabaseBusy`, extended +code 5, not capacity rejection: + +- production-profile zero-input Orchestrator restore; +- pending manual Worker claim restore reconciliation (500 instead of 409). + +Both failed alone too. `target/t748-busy-origin.log` captured the error and a +backtrace through `pin_internal_worker_restore_intent` -> the transaction helper +-> internal restore -> manual claim. The journal read authority first in a +DEFERRED transaction while the offloaded projection's other SQLite connection +could write. Such an upgrade can return BUSY without invoking SQLite's busy +handler; treating this as a retryable whole mutation would be incorrect. + +A new real-SQLite regression holds an IMMEDIATE writer on another connection, +registers an owned per-connection busy observer, and attempts one journal pin. +The old helper fails deterministically with BUSY/5 before waiting for ownership +(`target/t748-deferred-upgrade-red.log`, 0.23 seconds). The fixed helper waits +before its first authority read, while a current-thread timer and control task +progress. After releasing the writer it persists exactly one requested tuple; +there is no application-level mutation retry. + +`f597f1b5` makes this existing mutation helper explicitly +`with_immediate_transaction`. Restore journal and deletion authority reads, +mutation and commit remain in one transaction. Focused DB/HTTP tests (8), the +original restore and claim cases, and deletion tests (13) passed. + +## 4. Why a pre-dispatch failure could become the 600-second suite timeout + +The original long-running logging case joined an HTTP request with a fake provider +waiting for command dispatch. On pre-dispatch rejection the HTTP future can finish +with 503 while the receiver remains Pending: its channel is still owned by the +fixture, but no command will arrive. An unbounded `join!` waits forever in that +state and hides the actual error behind the outer suite timeout. + +The permanent `saturated_host_before_command_start_returns_availability_without_a_provider_operation` +regression uses the real signed HTTP route with a controlled full Host budget. It +observes the final 503/`db_capacity_exceeded` response and the still-live Pending +provider receiver, with no process or command dispatch. The original logging +fixture now asserts an early HTTP response before waiting for both branches, so +an unexpected rejection reports its body immediately rather than hanging. This +reproduces the causal timeout mechanism; the old log itself has no stack trace +from the hung provider future. + +The nine logging cases pass together at default parallelism (`49bfbfb2`). + +## Artifact/source isolation during comparison + +Initially the archived baseline used the existing Cargo target directory. That +replaced a same-package artifact/fingerprint; one later post-fix representative +command ran the baseline's 932-test binary (and `db_executor --list` had no tests). +That result was rejected as candidate evidence. All later candidate checks/tests +use `--target-dir target/t748-candidate-build`, separate from the baseline cache. +This changes artifact location, not test parallelism, features or admission policy. +The fresh candidate build listed the expected executor tests and total test count. + +## Validation and remaining composition boundary + +Pre-review code source `49bfbfb2` passed the full default-parallel Server suite, +including the command-overload case: 960 passed, zero failures, one existing +ignored migration fixture; 11 binary tests, library duration 175.29 seconds. +The only captured panics were the three deliberate unknown-outcome tests. +Root `cargo check` compile closure passed (including TUI/Server), as did Workdir +default (153 tests) and `http-client` (156 tests) full suites, format/diff checks +and state-contract names (558 matches, zero unapproved, zero stale exceptions). +These commands used the isolated candidate artifact directory; libtest default +parallelism and the production budgets were unchanged. The MR records the exact +published source, which also includes this report and the boundary document. + +## Post-publication independent review and causal fixes + +MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq3 independently reviewed published +source `66bb9d0b` and requested changes despite the successful original suite. +Its two major findings exposed boundaries the earlier tests did not prove: + +- `T748-DB-NETWORK-LANE`: mixed Subjektiv helpers could wait for synchronous + Runtime HTTP on the sole DB worker. Fix `d3fcfb29` places session read/search, + subject projection, memory staging, and legacy consolidation-scope helpers on + the existing finite Host owner; pure DB-resident operations keep DB placement. + Whole transaction helpers remain intact, and no Runtime I/O is introduced + inside a SQL transaction. +- `T748-ASYNC-PROTOCOL-SIGNING`: cold protocol source lookup called live binding + checks and signing directly on Tokio. Fix `c0223a0d` awaits the owned DB stage. + The source API now propagates failure explicitly: an invalid live binding or + signing/admission failure cannot be hidden as `None` or a legacy bearer fallback. + +Permanent real loopback HTTP regressions hold upstream Runtime replies during +session read/search, subject and legacy-scope requests, while independently +observing SQL, timer and control progress. Cancelling waiters retains the actual +Host lifecycle lease until completion. Cold protocol source tests control another +SQLite writer and shared connection mutex wait, and verify bounded overload, +fresh signing and revocation without fallback. Existing queue/cancellation and +deletion evidence now observes the actual mutation owner; the mixed fixtures +explicitly inject a small finite Host budget. Production budgets are unchanged. + +Focused boundary tests passed (10). Restoring the old placement produced a +failing regression, and fixed placement passed; an earlier syntax-rejected +instrumentation attempt is excluded from evidence. Diagnostic artifacts under +`target/` are not production source or replacement domain implementations. + +Exact code source `c0223a0d` then passed root `cargo check` and the original +**default-parallel** Server suite: **962 passed, zero failed, one pre-existing +ignored**, 11 binary tests; library duration **173.36 seconds**. Only the three +intentional executor panic diagnostics appeared, with no residual test processes. +Workdir default (153) and `http-client` (156) full suites, format/diff checks and +state-contract naming (558 matching lines, zero unapproved, zero stale) passed. +All used the isolated `target/t748-candidate-build` artifact directory. Logs: +`target/t748-review-fix-{root-check,full-suite,workdir-default,workdir-http,format,names}.log`. +The documentation-only publication `b10b7315` did not change that tested code. +MR seq5 independently confirmed the two old fixes and repeated the checks above, +but requested changes for a new mandatory-callback cycle; see the next section. + +## Mandatory callback must be independent of its waiting Host parent + +MR seq5 finding `T748-HOST-CALLBACK-CYCLE` traced Host create/restore -> Runtime +bootstrap -> mandatory Session attribution -> the same finite Host lane. Two +waiting parents could consume both workers while successfully admitted callbacks +could not run. Cancelling a waiter correctly retains the actual parent, so it is +not a solution. The Reviewer's compiled executor artifact proved this placement +cycle, not a complete create endpoint/E2E reproduction. + +Fix `a2487b57` separates store-only scope discovery into typed Subject, current +Job consolidation and legacy consolidation authority. Session attribution accepts +only Subject authority and executes its whole existing SQL operation on the +independent DB lane. It rejects consolidation roles without calling Runtime. +Mixed read/search, subject projections and allowed legacy consolidation operations +keep their Runtime checks on Host; the previous DB/network fix is not undone. + +The same dependency audit identified Subject resident context reads used by +Runtime bootstrap/restore. All Subject-only memory operations use store-only +Subject authority and DB ownership too (resident summary/context, proposal +validation, receipt lookup, explicit staging). Other memory operations retain +Host ownership because legacy authority can require synchronous Runtime lookup. +No transaction is split across an await and no production budget is increased. + +The strengthened signed route regression holds both actual production-sized Host +workers (2/16) in `run_host_workflow` parents and retains their create admission +leases until callbacks finish. Resident and attribution routes authenticate real +Runtime source proofs, check current singleton ownership, respond successfully, +and persist both distinct Subject session attributions **before** either parent +releases capacity. It also preserves Session callback reentry during deletion +drain. Runtime's wait is modeled at this policy boundary; no product process or +complete create endpoint is spawned. Separate denial coverage proves legacy +consolidation callbacks return 403 without Runtime lookup, leave no attribution, +and do not bypass authentication (401). + +Only Session attribution is an allowed deletion-drain callback; that endpoint +allowlist was not expanded. An initial fixture sent resident reads during deletion +and correctly received 409. The fixture now tests those reads before deletion, +then Session attribution while deletion waits. This is alignment with the existing +fence, not a relaxed product gate or success-picking retry. A diagnostic Python +invocation was unavailable before making any source change; only the subsequent +Deno lane-restoration runs are causal red evidence. + +Restoring just Session's old Host placement fails at its expected callback +watchdog; restoring just resident memory's old Host placement fails at its own +watchdog. Fixed placement succeeds without parent bailout. Evidence: +`target/t748-callback-old-{session,resident}-placement-red.log`, +`target/t748-callback-restored-green.log`. Final focused signed callback tests (5), +DB boundaries (10), and Subjektiv regressions (25) passed. + +Exact code `a2487b575df1f348251214c53da61595860fcb40` passed root `cargo check` and +original default-parallel Server **963 passed / 0 failed / 1 existing ignored**, +11 binary tests; library duration **173.01 seconds**. Only the three intentional +executor panics appeared, with no residual test processes. Workdir default153 / +http-client156, fmt/diff and naming558/0/0 passed. The isolated candidate cache +remains in use. Logs: `target/t748-callback-{root-check,full-suite,workdir-default,workdir-http}.log`. +The docs-only publication `27b6c688` changed no tested code. MR seq7 independently +confirmed the callback fix and earlier fixes, but identified another direct async +caller; its independent verdict was `request_changes`, not approval. + +## Worker discovery and adjacent metadata caller audit + +MR seq7 major `T748-ASYNC-WORKER-DISCOVERY` traced async discovery directly into +synchronous `workers_response`. That helper acquires the shared SQL connection +for the Worker projection snapshot, Workdir registry/links and authority projection. +The Reviewer's independently compiled public snapshot helper reproduced a +300.246ms shared-mutex stall while its Tokio timer could not progress. This was +helper-boundary evidence, not an authenticated endpoint/E2E run. + +Fix `dbcea6c3` awaits the **whole existing** `workers_response` on the independent +bounded DB owner. Source-proof requirement, Workspace/query/cursor validation, +visibility filtering, pagination and control-grant behavior are unchanged. The +other production callers were traced: Worker list/resource detail already use DB; +mixed Runtime cleanup plans use Host. No network work is added to the DB owner. + +The adjacent metadata audit found async `workspace_runtime_resources_response` +also directly calling synchronous Runtime summaries (including remote signed HTTP +and its awaited DB authorization). This call now uses finite Host ownership, +then the existing independent async binding read. Runtime list/detail share this +boundary. Whole SQL helpers, admission budgets and authentication fallback rules +are not changed. + +The permanent post-auth discovery handler regression uses a separate SQLite +IMMEDIATE writer and busy observer to hold the actual shared connection mutex. +While the accepted discovery read is queued, current-thread timer and independent +Host control progress. A second read receives 503/`db_capacity_exceeded`; a source +without proof receives 403 before admission. Cancellation retains the actual +queued read's lease/budget until result cleanup; ordinary completion preserves the +empty filtered catalog response. Signed route proof verification remains covered +by the existing dedicated-permission test, separately from this post-auth policy +boundary so middleware's own DB wait cannot hide the handler bug. + +Restoring the exact old discovery call fails its first-poll assertion after the +fixture watchdog releases the synchronous mutex wait. Restoring only the old +Runtime-summary placement fails because Tokio cannot observe the real loopback +request until the upstream watchdog releases it. Fixed summary placement lets +SQL/timer/control progress during the held signed upstream request and keeps the +actual Host lease after cancellation. Explicit upstream-release observation +prevents watchdog failure cleanup from masquerading as progress. Logs: +`target/t748-discovery-old-placement-red.log`, +`target/t748-runtime-summary-old-placement-red.log`, +`target/t748-discovery-summary-fixed-boundaries.log` (12 passed), +`target/t748-discovery-summary-policy.log` (4), and callbacks (5). + +Exact code `dbcea6c34ba55453ce588d68272df238cde19f47` passed root `cargo check` and +original default-parallel Server **965 passed / 0 failed / 1 existing ignored**, +11 binary tests; library duration **175.00 seconds**. Only three deliberate +executor panics appeared, with no residual test processes. Workdir default153 / +http-client156 and fmt/diff/naming558/0/0 passed. Logs use the same isolated +candidate cache: `target/t748-discovery-summary-{root-check,full-suite,workdir-default,workdir-http}.log`. +Docs-only publication `b63bdb44` changed no tested code. MR seq9 independently +confirmed these fixes, then requested changes for Runtime management's residual +synchronous authority calls; it was not an approval of the complete source. + +## Runtime management: authority, network, and accepted publication ownership + +Seq9 major `T748-ASYNC-RUNTIME-MANAGEMENT` identified registration's direct +`get_validated` signing identity read and post-ping direct live-binding fence. +The Reviewer reproduced both exact compiled helpers waiting approximately 300ms +on the shared SQL mutex with no Tokio timer progress (helper-boundary, not endpoint +or E2E proof). Registration's synchronous Runtime Worker scan also requires a +network/authority boundary; ping previously used an unbounded blocking admission. + +Fix `925bce2a` introduces whole owned DB stages for validated active signing +identity and the post-ping current binding fence. Runtime Worker scans and ping +use existing finite Host admission. A changed/revoked binding still produces +conflict rather than attesting the earlier connection; owner/key/CAS validation +and existing whole store transactions are retained. + +The adjacent management audit also closes cancellation separation between SQL +commit and local Runtime publication. Registration and metadata updates now own +mutation, expectations/registry/subscription publication, and response observation +in one accepted Host workflow, with their SQL helper awaited on the independent +DB owner. Public Runtime resource/detail helpers admit ownership; the already-owned +workflows call explicit `_on_host` variants directly instead of reacquiring the +same lane. There is no Runtime/network await inside a SQL transaction or nested +Host admission. A cancelled waiter cannot strand a committed metadata change +without publishing the matching binding. + +Launch options were another mixed Runtime summary/config helper placed on sole +DB. Its signed Runtime authorization could enqueue DB work behind its own waiting +parent. It now uses Host, preserving SQL/control progress and finite ownership. +The audit traced remaining signing validation/binding-fence calls: UI signing +operations already use DB, Runtime gate calls occur on Host, and removal already +has an owned Host workflow. DNS validation stays async; pure local registration +construction/notification stays in the accepted publication owner. + +Permanent regressions cover both authority stages under a real separate SQLite +IMMEDIATE writer and shared-mutex busy observer: current-thread timer/control +progress, overload503, cancelled actual DB lease retention/cleanup, positive +validation and revoked binding409. An actual signed loopback ping is held while +SQL contention starts; after its reply the real handler's post-ping fence queues +without blocking Tokio or Host control. Another signed loopback scan proves SQL/ +timer progress and retained cancelled finite Host admission. The metadata mutation +case cancels while its own SQL is waiting, then observes one durable update and +an equal published binding after actual Host/DB drain. Producers are stopped and +drained while the original fixture runtime remains alive. + +Old identity, binding, scan and launch placements each fail their expected +contention/watchdog boundary; fixed code passes. Logs: +`target/t748-management-old-{identity,binding,scan,launch}-placement-red.log`, +`target/t748-management-final-restored-green.log` (16 boundaries), registration +policy11 and connection policy13. A zero-match launch-options exploration is not +counted as proof; its real held-summary case is in the 16 boundaries. The first +fixture teardown named an unavailable method and failed compilation; it was +corrected to the existing `shutdown_async`, not counted as a semantic test run. + +Exact code `925bce2a405b6d3255887fba88867ab8d23882dc` passed root `cargo check` and +original default-parallel Server **969 passed / 0 failed / 1 existing ignored**, +11 binary tests; library duration **172.25 seconds**. Only three deliberate +executor panics appeared, with no residual test processes. Workdir default153 / +http-client156, fmt/diff/naming558/0/0 passed. Same isolated cache; evidence logs: +`target/t748-management-{root-check,full-suite,workdir-default,workdir-http}.log`. +Docs-only publication `6d8c49b7` changed no tested code. MR seq11 independently +confirmed those fixes and repeated the full validation, but requested changes for +cancelled trust-key revocation; it did not approve the complete snapshot. + +## Trust-key revocation must own post-commit subscription termination + +Seq11 major `T748-RUNTIME-REVOKE-CANCEL-PUBLICATION` reproduced the authenticated +public DELETE trust-key route waiting for an actual SQLite writer. Cancelling its +HTTP waiter did not cancel accepted SQL: revoke, binding-ID replacement and audit +committed, but waiter-owned expectation removal and broker unregister were lost. +The independent artifact used the normal public Server library's inner router, +not outer Workspace dispatch or live-stream/E2E. New signed requests still fail +the live binding fence; that does not shut down an already connected subscription. + +Fix `1a065db0` gives the whole key-revoke outcome one accepted finite Host owner: +intact CAS/audit SQL on the independent DB lane, local expectation removal, broker +unregister/Shutdown, and response observation. Owner/scope/input preflight, typed +trust conflicts and current-binding error responses remain unchanged. Response +reads use the already-owned `_on_host` detail helper, with no nested Host admission. +Admission/unknown-outcome errors retain the normal typed availability mapping; +there is no transaction split, new queue budget or automatic mutation retry. +The adjacent key-mutation caller audit confirms registration and metadata already +own publication and this is the sole production key-revoke caller. + +The permanent cancellation regression exercises the **real outer Workspace +dispatch**, owner browser cookie, generated DELETE route, whole store mutation and +broker. Projection producers stop before the write lock is acquired, so the +fixture-local busy observer identifies this actual revoke transaction, not startup +SQL. Timer/control progress before the watchdog releases SQLite. After waiter +cancellation, the real admitted RequestLease keeps deletion Pending. Releasing the +writer and draining actual Host/DB owners persists exactly one revoke audit, +changes binding ID, removes expectations and broker registration, and delivers +`Closed/ServerShutdown` to the downstream. The loopback endpoint holds a WebSocket +handshake; this proves broker termination, not an established upstream socket or +complete product E2E. Existing broker protocol tests cover socket replacement. + +A separate full-Host route regression observes 503 before mutation and unchanged +binding, audit and publication. Restoring only the exact old revoke method fails +both permanent tests: the cancelled job leaves broker registration behind, and +under overload the old handler commits revoke before its response helper returns +503. Fixed code passes both without retry or reduced parallelism. Evidence: +`target/t748-revoke-final-old-publication-red.log`, +`target/t748-revoke-fixed-boundaries.log` (18), `target/t748-revoke-policy.log` (3), +`target/t748-revoke-binding-policy.log` (16). +An initial counter probe outside dispatch was invalid because dispatch replaces it +with the real RequestLease; it was corrected to observe actual deletion drain. +A test-only overload channel type-inference compile rejection was corrected with +an explicit unit channel. Neither setup error is counted as semantic evidence. + +Exact code `1a065db0fc5d07e2dcc1b9dac2fcfc0607b9a23c` passed root `cargo check` and +the original default-parallel Server **971 passed / 0 failed / 1 existing ignored**, +11 binary tests; library duration **173.05 seconds**. Only three deliberate executor +panic diagnostics appeared, and no retained test/repro processes were observed. +Workdir default153/http-client156, fmt/diff and naming558/0/0 passed. All Cargo +checks/tests use the isolated candidate cache. Logs: +`target/t748-revoke-{root-check,full-suite,workdir-default,workdir-http,fmt,names}.log`. +Docs-only source `ebce7ab7` changed no tested code. MR seq13 independently confirmed +the revoke ownership fix and repeated the full checks, but requested changes for +Ticket cancellation, selector repair and post-restore projection's async callers. +The current uncommitted follow-up and the requested comprehensive caller audit +are tracked in [the caller inventory](2026-10-12-t748-caller-inventory.md). +Successful suites are not evidence that uninspected caller boundaries are safe. + +No latency benchmark, production incident measurement, target integration, +Ticket conclusion, or running-environment update is claimed. T743 has now +published its caller inventory/source; its owner will compose the T748 facade and +new callers, then obtain review of that exact combined source. T748 does not copy +those domain APIs or wait on the former unpublished branch. diff --git a/docs/report/2026-10-12-t748-t743-composition.md b/docs/report/2026-10-12-t748-t743-composition.md new file mode 100644 index 000000000..b5b94b67a --- /dev/null +++ b/docs/report/2026-10-12-t748-t743-composition.md @@ -0,0 +1,34 @@ +# T748 / T743 orchestrated composition + +## Exact inputs and ownership + +- Integrated standalone T743 target: `05ba6a43adafc2a74fac29295b906bcb30f83f11`. +- Independently approved T748 source: `e436b2ae58e940db416a5d6a9c2edb1368ec7600`. +- Shared original develop base: `56795fb92cb9def9bcd701bd51d92e99526f8eff`. + +T743's corrective revert deliberately removed the prematurely imported T748 tree while preserving published history. Ordinary merge therefore reports historical modify/delete conflicts; explicit composition against the original develop base restores the approved T748 platform while retaining standalone T743's public APIs/UI/Tools/Prompt. The resulting commit has both real input parents: no history rewrite, substituted source candidate, or change to T743's concluded requirements. The Orchestrator owns this composition and target integration; T743 is not reopened or gated on another lane. + +## New caller execution boundaries + +- `ticket_assignments_page` and `ticket_assignment_diagnostics`: owned clones and reference/scope lookup plus the whole existing synchronous query helper run on the shared finite DB executor. +- `ticket_assignment_answer`: mixed Runtime source discovery, intact source/CAS/replay/audit/receipt/commit helper, and post-commit orchestrator discovery/delivery run under one accepted finite Host owner. Runtime/network waits never occupy the sole DB worker. The transaction is not divided by awaits, nested Host admission is not introduced, and request cancellation does not cancel/replay accepted work or lose commit-before-publication responsibility. +- Browser forwarders remain delegates to those canonical methods; no second domain authority or DTO is introduced. + +## Orchestrator validation + +Commands used the repository's original default parallelism and locked dependencies: + +- Root `cargo check --locked`: pass. +- Assignment tracking library suite: 28 pass, including public answer source/replay/CAS/reassignment and page/cursor authorization semantics. +- DB execution regressions: 36 pass (the 33 approved upstream tests plus 3 new public caller tests). +- Complete `yoi-workspace-server --all-targets`: library 1007 pass / 0 fail / 1 pre-existing ignored; binary 11 pass. Library completed in 189.81 seconds. +- `workdir --all-targets`: 153 pass; with `http-client`: 156 pass. +- Root format check and staged/unstaged diff checks: pass. + +New public-caller tests use the real generated Browser handlers on a current-thread Tokio runtime, explicit isolated finite executor budgets, and a deterministic actual owner/connection hold. After job admission, independent control and timer progress, cancellation leaves the queued slot occupied, overload is typed HTTP 503, drain waits for actual completion, and subsequent normal lookup has reusable capacity. They do not claim successful answer commit or Runtime E2E delivery; the existing domain and real SQLite regressions cover those distinct boundaries. + +Causal red/green: replacing only the three new adapters with their standalone T743 synchronous bodies made all 3 new tests fail after the bounded 5-second watchdog released the held connection; restoring the finite adapters made all 3 pass. Both logs remain in the integration Workdir's `target/t748-integration-public-boundary{,-red}.log`. Complete suite, DB, and Workdir logs use the same `target/t748-integration-*` prefix. An initial Workdir command used the nonexistent package name `yoi-workdir` and failed selection before running tests; the correct Cargo package is `workdir`, as subsequently validated. No failed test was retried to pick a green result. + +## Review and integration status at publication + +This is an Orchestrator-authored candidate for exact-source independent review, not a claim that prior T748 approval covers this new composition. The stable `work/T-748-orchestrator-composition` ref publishes the merge candidate without updating develop. The existing assigned T748 Coder receives it for a normal merge into the same immutable MR source selector and owns the trusted Reviewer/fix loop. Only the Orchestrator will update develop after fresh exact-source readiness. No replacement MR, new Reviewer Runtime Worker, target integration, live deployment, Ticket conclusion, or resource cleanup is claimed here. diff --git a/docs/report/2026-10-12-worker-observation-followup-gap.md b/docs/report/2026-10-12-worker-observation-followup-gap.md new file mode 100644 index 000000000..fe5ef448a --- /dev/null +++ b/docs/report/2026-10-12-worker-observation-followup-gap.md @@ -0,0 +1,49 @@ +# Internal SubWorker follow-up observation gap + +## Observed barrier + +During T748's read-only caller inventory, the parent could read the first Runtime +inventory assistant result by stable entry ref +`E01a12e76-3893-70a0-bb73-28ee0c52f87b`. It then sent a follow-up to the same +Idle direct child, `t748-runtime-caller-inventory`, to close four remaining +execution/ownership boundaries. + +After a turn-ended/Idle notification, `WorkerList` confirmed Idle, but bounded +`SearchSessionEntries` for assistant entries after follow-up user ref +`E01a12e76-d696-73d3-80d7-b8cde9935b37` returned no entries. Tool observations +were visible. The overview showed only the initial and first follow-up user +anchors, not a final result. A single additional Submit asking for the existing +findings (not re-investigation) returned success. After its turn-ended/Idle +notification, the same bounded assistant query still returned no entries, and the +overview still ended at the first follow-up user anchor. + +This is an observation-surface symptom, **not proof** that an assistant result +was generated, lost, or that the underlying investigation completed. No raw +session storage or hidden worker state was read to bypass the projection. + +The same symptom also occurred for `t748-domain-caller-inventory`: its first +result `E01a12e73-7ae4-7f02-9a7e-203140971635` was readable, but after the +follow-up turn-ended/Idle notification the overview ended at the follow-up's +starting commentary `E01a12e75-9c19-7bd0-9284-299116a05761` and intervening tool +records. No final follow-up result was visible. The parent released that +read-only delegation without another Submit and took over the remaining generated +adapter/forwarder and launch lower-edge checks. Neither `WorkerStop` tool outcome +was treated as evidence that the investigation completed. + +## Handling + +The parent did not mark the delegated work or Ticket complete, publish source, +or replace independent review with the notification. It stopped only the two +parent-owned read-only children to release delegation and took over the open +Runtime and domain boundaries. No live product process, parent Worker, target +branch, or attachment was stopped or cleaned. + +## Improvement proposal + +Expose enough committed turn metadata through the authorized observation surface +to distinguish a finalized assistant result, a turn ending without a result, and +an observation capture not yet including an accepted Submit. An explicit +completion marker tied to the visible Submit entry would prevent an Idle +notification from being confused with inspectable evidence. Preserve append-only +session entries and the current authorization projection; do not expose raw +prompts/reasoning or require polling/retry loops as a workaround.