From c6d73e8286d743e0329154d49b52d4bee53e8d60 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 12 Oct 2026 15:03:44 +0900 Subject: [PATCH] revert: remove withdrawn T748 composition from T743 source Mainline corrective revert of 5c5727e3; preserve published history and report bodies. Implementation, public contracts and T743 semantic tests return exactly to 9b0d42fd against develop56795fb9. T748 platform, only-dependent adapter and execution fixtures are not T743 return gates. Independent T743 review and fixes continue on this same branch/MR. --- crates/workdir/src/http.rs | 161 +- .../src/assignment_delivery_tests.rs | 16 +- .../src/assignment_tracking_tests.rs | 18 +- crates/workspace-server/src/auth.rs | 202 +- crates/workspace-server/src/authority.rs | 15 +- crates/workspace-server/src/config_source.rs | 12 +- crates/workspace-server/src/db_executor.rs | 957 -- .../src/frozen_content_state_tests.rs | 2 - .../src/frozen_repository_key_tests.rs | 4 - .../src/frozen_runtime_binding_tests.rs | 4 +- crates/workspace-server/src/hosts.rs | 145 +- .../src/hosts/tests/workspace_auth_tests.rs | 3 +- .../src/legacy_evidence_column_tests.rs | 4 +- crates/workspace-server/src/lib.rs | 7 - crates/workspace-server/src/memory_staging.rs | 2 +- .../src/migration_entrypoint_tests.rs | 6 +- .../workspace-server/src/repository_access.rs | 2 +- crates/workspace-server/src/retention.rs | 2 +- .../src/runtime_subscription.rs | 13 +- .../src/schema_equivalence_tests.rs | 4 +- crates/workspace-server/src/server.rs | 8606 +++++++---------- .../src/server/subjektiv_job.rs | 148 +- .../src/server/tests/api_logging_tests.rs | 89 +- .../src/server/tests/auth_logging_tests.rs | 8 +- .../src/server/tests/drive_tests.rs | 6 +- .../server/tests/drive_web_roundtrip_tests.rs | 4 +- .../tests/protocol_backpressure_tests.rs | 5 +- .../src/server/tests/subject_spawn_tests.rs | 2 +- .../tests/ticket_worker_launch_tests.rs | 2 +- .../src/server/tests/worker_drive_tests.rs | 2 +- .../server/tests/worker_operations_tests.rs | 15 +- .../server/tests/workspace_admission_tests.rs | 60 +- .../src/server/worker_operations.rs | 358 +- .../src/server/workspace_admission.rs | 22 - .../src/server_assignment_attention_tests.rs | 2 +- .../server_assignment_db_execution_tests.rs | 1408 --- .../src/server_assignment_delivery_tests.rs | 4 +- .../src/server_db_execution_tests.rs | 769 -- crates/workspace-server/src/server_drive.rs | 13 +- .../src/server_drive_grants.rs | 73 +- .../src/server_ticket_case_signed_tests.rs | 2 +- .../src/server_ticket_tags_tests.rs | 2 +- .../src/server_workspace_config.rs | 433 +- .../src/server_workspace_config_tests.rs | 8 +- .../src/server_workspace_config_wip_tests.rs | 2 +- crates/workspace-server/src/store.rs | 881 +- .../src/ticket_tag_upgrade_tests.rs | 4 +- .../src/workdir_create_operations.rs | 2 +- .../workspace-server/src/workdir_removal.rs | 2 +- .../workspace-server/src/worker_projection.rs | 455 +- crates/workspace-server/src/worker_source.rs | 42 +- .../src/worker_source/runtime_request.rs | 410 +- .../workspace-server/src/workspace_catalog.rs | 28 +- .../src/workspace_deletion.rs | 14 +- .../src/workspace_drive_host.rs | 73 +- .../src/workspace_signing_identity.rs | 6 +- .../src/workspace_subscription.rs | 222 +- docs/development/backend-db-execution.md | 115 - .../report/2026-10-12-t748-suite-diagnosis.md | 166 - 59 files changed, 4222 insertions(+), 11820 deletions(-) delete mode 100644 crates/workspace-server/src/db_executor.rs delete mode 100644 crates/workspace-server/src/server_assignment_db_execution_tests.rs delete mode 100644 crates/workspace-server/src/server_db_execution_tests.rs delete mode 100644 docs/development/backend-db-execution.md delete mode 100644 docs/report/2026-10-12-t748-suite-diagnosis.md diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index ae155b008..940506ca7 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -432,23 +432,16 @@ mod client { use super::*; use crate::{Workdir, WorkdirSession}; - /// An owned, Send authorization future that can offload synchronous authority - /// work without borrowing request data or the authorization provider. - pub type WorkdirHttpAuthorizationFuture = std::pin::Pin< - Box> + Send + 'static>, - >; - - /// Provides a fresh bearer token asynchronously for each Runtime request. - /// Backend implementations can mint short-lived capability tokens without - /// making a Worker-bound session expire with the token used to open it. - /// Blocking DB/signing work must be offloaded by the implementation. + /// Provides a fresh bearer token for each Runtime request. Backend + /// implementations can mint short-lived capability tokens without making a + /// Worker-bound session expire with the token used to open it. pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync { fn bearer_token( &self, - method: String, - path_and_query: String, - body: Vec, - ) -> WorkdirHttpAuthorizationFuture; + method: &str, + path_and_query: &str, + body: &[u8], + ) -> Result; } struct FixedBearerToken(Arc); @@ -462,12 +455,11 @@ mod client { impl WorkdirHttpAuthorization for FixedBearerToken { fn bearer_token( &self, - _method: String, - _path_and_query: String, - _body: Vec, - ) -> WorkdirHttpAuthorizationFuture { - let token = self.0.clone(); - Box::pin(async move { Ok(token.to_string()) }) + _method: &str, + _path_and_query: &str, + _body: &[u8], + ) -> Result { + Ok(self.0.to_string()) } } @@ -517,9 +509,7 @@ mod client { )?; let body = serde_json::to_vec(&request) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = authorization - .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) - .await?; + let token = authorization.bearer_token("POST", url.path(), &body)?; let response = client .post(url) .bearer_auth(token) @@ -570,10 +560,7 @@ mod client { let operation = WorkdirSessionOperationRequest { operation }; let body = serde_json::to_vec(&operation) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = self - .authorization - .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) - .await?; + let token = self.authorization.bearer_token("POST", url.path(), &body)?; let response = self .client .post(url) @@ -877,10 +864,7 @@ mod client { &self.base_url, &["v1", "workdir-sessions", self.session_id.as_str()], )?; - let token = self - .authorization - .bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new()) - .await?; + let token = self.authorization.bearer_token("DELETE", url.path(), &[])?; let response = self .client .delete(url) @@ -934,124 +918,11 @@ mod client { WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string()) } - #[cfg(test)] - mod authorization_tests { - use super::*; - use std::sync::Mutex; - - #[derive(Debug, Default)] - struct RejectAuthorization { - requests: Arc)>>>, - } - - impl WorkdirHttpAuthorization for RejectAuthorization { - fn bearer_token( - &self, - method: String, - path_and_query: String, - body: Vec, - ) -> WorkdirHttpAuthorizationFuture { - let requests = self.requests.clone(); - Box::pin(async move { - tokio::task::yield_now().await; - let mut requests = requests.lock().unwrap(); - requests.push((method, path_and_query, body)); - Err(WorkdirError::Unavailable(format!( - "authorization rejected request {}", - requests.len() - ))) - }) - } - } - - #[tokio::test] - async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() { - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); - let base_url: Url = format!("http://{}", listener.local_addr().unwrap()) - .parse() - .unwrap(); - let client = Client::builder().no_proxy().build().unwrap(); - let authorization = Arc::new(RejectAuthorization::default()); - let workdir = Workdir::new("w"); - let open_request = OpenWorkdirSessionRequest { - owner_worker_id: Some("worker".to_string()), - }; - let error = RemoteWorkdirSession::open_with_authorization( - client.clone(), - base_url.clone(), - authorization.clone(), - workdir.id().clone(), - open_request.clone(), - ) - .await - .unwrap_err(); - assert!(matches!( - error, - WorkdirError::Unavailable(message) if message == "authorization rejected request 1" - )); - - // Seed an already-open session to exercise the public operation and - // close boundaries independently of the rejected open request. - let remote = RemoteWorkdirSession { - client, - base_url, - authorization: authorization.clone(), - workdir, - session_id: WorkdirSessionId::new("s").unwrap(), - capabilities: WorkdirSessionCapabilities::EMPTY, - closed: AtomicBool::new(false), - }; - let stat = StatRequest { - path: crate::WorkdirPath::new("file").unwrap(), - }; - let error = remote.stat(stat.clone()).await.unwrap_err(); - assert!(matches!( - error, - WorkdirError::Unavailable(message) if message == "authorization rejected request 2" - )); - let error = remote.close().await.unwrap_err(); - assert!(matches!( - error, - WorkdirError::Unavailable(message) if message == "authorization rejected request 3" - )); - assert_eq!( - *authorization.requests.lock().unwrap(), - vec![ - ( - "POST".to_string(), - "/v1/working-directories/w/sessions".to_string(), - serde_json::to_vec(&open_request).unwrap(), - ), - ( - "POST".to_string(), - "/v1/workdir-sessions/s/operations".to_string(), - serde_json::to_vec(&WorkdirSessionOperationRequest { - operation: WorkdirSessionOperation::Stat(stat), - }) - .unwrap(), - ), - ( - "DELETE".to_string(), - "/v1/workdir-sessions/s".to_string(), - Vec::new(), - ), - ] - ); - assert_eq!( - listener.accept().unwrap_err().kind(), - std::io::ErrorKind::WouldBlock - ); - } - } - pub use self::RemoteWorkdirSession as ClientSession; } #[cfg(feature = "http-client")] -pub use client::{ - ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture, -}; +pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization}; #[cfg(test)] mod tests { diff --git a/crates/workspace-server/src/assignment_delivery_tests.rs b/crates/workspace-server/src/assignment_delivery_tests.rs index c8e2cb7f6..b8702edb7 100644 --- a/crates/workspace-server/src/assignment_delivery_tests.rs +++ b/crates/workspace-server/src/assignment_delivery_tests.rs @@ -147,7 +147,7 @@ async fn assignment_delivery_response_loss_returns_immutable_payload_after_progr .update_ticket_assignment_tracking("w", "tracking", &update, &source) .unwrap(); drop(store); - let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); assert_eq!( store .list_pending_ticket_assignment_deliveries("w", 100) @@ -208,7 +208,7 @@ async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt() let path = directory.path().join("workspace.db"); let input = pending.input_event_id.clone(); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open_for_test(path).unwrap(); + let store = SqliteWorkspaceStore::open(path).unwrap(); barrier.wait(); store .reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source()) @@ -568,7 +568,7 @@ async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_co "original rendered prompt" ); drop(store); - let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); assert_eq!( store .prepare_ticket_assignment_delivery_content( @@ -634,7 +634,7 @@ fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_l let operations = stored_fixture_rows(&conn, "ticket_assignment_operations"); let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history"); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let pending = store .list_pending_ticket_assignment_deliveries("w", 100) .unwrap(); @@ -684,7 +684,7 @@ async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once let barrier = barrier.clone(); let path = directory.path().join("workspace.db"); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open_for_test(path).unwrap(); + let store = SqliteWorkspaceStore::open(path).unwrap(); barrier.wait(); delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source()) .unwrap() @@ -716,8 +716,8 @@ fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() { let conn = tracking_schema91_fixture(&path); migrate_ticket_tracking_v91_to_v92(&conn).unwrap(); drop(conn); - let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory().unwrap(); let objects = |store: &SqliteWorkspaceStore| { store.with_conn(|conn| { let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?; @@ -741,7 +741,7 @@ async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() { .unwrap(); drop(store); assert!( - matches!(SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) + matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) ); } diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 5255267c7..513daef26 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -78,7 +78,7 @@ async fn tracking_store_every_replay_fences_current_singleton_and_authenticated_ ); // Model the authority change after authenticated source capture, before // BEGIN IMMEDIATE on a second connection. Receipt presence cannot bypass it. - let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); + let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); second.with_conn_mut(|conn| { let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; match change { @@ -233,7 +233,7 @@ async fn tracking_store_same_principal_reissue_and_exact_retries_keep_original_r .is_err() ); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); assert_eq!( reopened .get_ticket_assignment_tracking("w", "tracking", "a") @@ -441,7 +441,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .collect::>(), vec![92, 93] ); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, 93); @@ -525,7 +525,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .list_ticket_assignment_tracking_history("w", "pending", "a-pending") .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); assert_eq!( reopened .list_ticket_assignment_tracking_history("w", "pending", "a-pending") @@ -550,7 +550,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first let releases_before = stored_fixture_rows(&conn, "ticket_assignment_work_releases"); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "w").unwrap(); let reopen = || { let observed = backend.show("historical-done".into()).unwrap(); @@ -645,7 +645,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first ); // A second observer holds the same Assign baseline. Only the first // explicit writer may consume it; BEGIN IMMEDIATE serializes the CAS. - let second = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let second = SqliteWorkspaceStore::open(&path).unwrap(); assert_eq!( second .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") @@ -740,7 +740,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first }).unwrap(); drop(second); drop(store); - let reopened_store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let reopened_store = SqliteWorkspaceStore::open(&path).unwrap(); assert_eq!( reopened_store .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") @@ -796,7 +796,7 @@ fn tracking_store_schema91_missing_assign_audit_remains_unknown_without_cas_base ) .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let unknown = store .get_ticket_assignment_tracking("w", "historical-done", "a-historical-done") .unwrap() @@ -1068,7 +1068,7 @@ async fn orchestrator_assignment_all_runtime_retries_fence_singleton_and_binding &source, ) .unwrap(); - let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap(); + let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); second.with_conn_mut(|conn| {let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;match change { "replace" => {tx.execute("UPDATE worker_singleton_owners SET worker_id='replacement',generation=2 WHERE workspace_id='w'",[])?;}, "revoke" => {tx.execute("UPDATE workspace_runtime_bindings SET revoked_at='revoked' WHERE workspace_id='w'",[])?;}, diff --git a/crates/workspace-server/src/auth.rs b/crates/workspace-server/src/auth.rs index b5badef0d..aa202f2de 100644 --- a/crates/workspace-server/src/auth.rs +++ b/crates/workspace-server/src/auth.rs @@ -89,30 +89,12 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option { }) } -pub async fn resolve_request_actor( - store: std::sync::Arc, +pub async fn resolve_request_actor( + store: &S, headers: &HeaderMap, cookie_name: &str, ) -> Result> { - let bearer = parse_bearer(headers); - let session = parse_cookie(headers, cookie_name); - if bearer.is_none() && session.is_none() { - return Ok(None); - } - let executor = store.db_executor(); - executor - .run(move || resolve_request_actor_sync(store.as_ref(), bearer, session)) - .await -} - -// One owned synchronous stage includes token usage marking and the user read. -// Do not call async store wrappers here: they use the same executor themselves. -fn resolve_request_actor_sync( - store: &dyn ControlPlaneStore, - bearer: Option, - session: Option, -) -> Result> { - if let Some(token) = bearer { + if let Some(token) = parse_bearer(headers) { let token_hash = token_hash(&token); if let Some(api_token) = store.resolve_api_token(&token_hash)? { if api_token @@ -127,7 +109,7 @@ fn resolve_request_actor_sync( } } - if let Some(session_token) = session { + if let Some(session_token) = parse_cookie(headers, cookie_name) { if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? { if is_expired(&session.expires_at) { return Ok(None); @@ -191,182 +173,6 @@ pub fn auth_error(code: &str, message: &str) -> Error { #[cfg(test)] mod tests { use super::*; - use crate::store::{ - AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord, - }; - use std::sync::Arc; - - fn seeded_store() -> Arc { - let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { - workers: std::num::NonZeroUsize::new(1).unwrap(), - queued: std::num::NonZeroUsize::new(1).unwrap(), - }) - .unwrap(); - let store = Arc::new( - SqliteWorkspaceStore::in_memory_for_test() - .unwrap() - .with_db_executor(executor), - ); - store - .upsert_account(&AccountRecord { - account_id: "account-auth".into(), - kind: "user".into(), - handle: "auth".into(), - display_name: "Auth".into(), - created_at: "1".into(), - updated_at: "1".into(), - }) - .unwrap(); - store - .upsert_user(&UserRecord { - user_id: "user-auth".into(), - account_id: "account-auth".into(), - handle: "auth".into(), - display_name: "Auth".into(), - created_at: "1".into(), - updated_at: "1".into(), - }) - .unwrap(); - store - .create_browser_session(&BrowserSessionRecord { - session_id: "session-auth".into(), - token_hash: token_hash("session-secret"), - user_id: "user-auth".into(), - created_at: "1".into(), - expires_at: "9999-01-01T00:00:00Z".into(), - revoked_at: None, - }) - .unwrap(); - store - } - - #[tokio::test] - async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() { - let store = seeded_store(); - let hash = token_hash("api-secret"); - store - .create_api_token(&ApiTokenRecord { - token_id: "token-auth".into(), - token_hash: hash.clone(), - user_id: "user-auth".into(), - label: "Auth".into(), - created_at: "1".into(), - expires_at: None, - revoked_at: None, - last_used_at: None, - }) - .unwrap(); - let mut headers = HeaderMap::new(); - headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap()); - headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); - let actor = resolve_request_actor(store.clone(), &headers, "session") - .await - .unwrap() - .unwrap(); - assert_eq!(actor.user_id, "user-auth"); - assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken); - assert!( - store - .resolve_api_token(&hash) - .unwrap() - .unwrap() - .last_used_at - .is_some() - ); - } - - #[tokio::test] - async fn expired_api_token_does_not_fall_back_to_valid_browser_session() { - let store = seeded_store(); - let hash = token_hash("expired-secret"); - store - .create_api_token(&ApiTokenRecord { - token_id: "token-auth".into(), - token_hash: hash.clone(), - user_id: "user-auth".into(), - label: "Auth".into(), - created_at: "1".into(), - expires_at: Some("2000-01-01T00:00:00Z".into()), - revoked_at: None, - last_used_at: None, - }) - .unwrap(); - let mut headers = HeaderMap::new(); - headers.insert( - header::AUTHORIZATION, - "Bearer expired-secret".parse().unwrap(), - ); - headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); - assert!( - resolve_request_actor(store.clone(), &headers, "session") - .await - .unwrap() - .is_none() - ); - assert!( - store - .resolve_api_token(&hash) - .unwrap() - .unwrap() - .last_used_at - .is_none() - ); - } - - #[tokio::test] - async fn unknown_bearer_can_fall_back_to_valid_browser_session() { - let store = seeded_store(); - let mut headers = HeaderMap::new(); - headers.insert( - header::AUTHORIZATION, - "Bearer unknown-secret".parse().unwrap(), - ); - headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); - let actor = resolve_request_actor(store, &headers, "session") - .await - .unwrap() - .unwrap(); - assert_eq!(actor.user_id, "user-auth"); - assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession); - } - - #[tokio::test] - async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() { - let store = seeded_store(); - let executor = store.db_executor(); - let (started, started_rx) = tokio::sync::oneshot::channel(); - let (release, release_rx) = std::sync::mpsc::channel(); - let blocker_executor = executor.clone(); - let blocker = tokio::spawn(async move { - blocker_executor - .run(move || -> Result<()> { - let _ = started.send(()); - release_rx.recv().unwrap(); - Ok(()) - }) - .await - }); - tokio::time::timeout(std::time::Duration::from_secs(1), started_rx) - .await - .unwrap() - .unwrap(); - let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) })); - assert!(futures::poll!(queued.as_mut()).is_pending()); - let mut headers = HeaderMap::new(); - headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); - let result = tokio::time::timeout( - std::time::Duration::from_secs(1), - resolve_request_actor(store, &headers, "session"), - ) - .await; - release.send(()).unwrap(); - blocker.await.unwrap().unwrap(); - queued.await.unwrap(); - assert!(matches!( - result.expect("capacity rejection must not wait for queued SQL"), - Err(Error::DbCapacityExceeded) - )); - } #[test] fn user_code_uses_the_uuid_random_tail() { diff --git a/crates/workspace-server/src/authority.rs b/crates/workspace-server/src/authority.rs index f52ca0f20..1203f2503 100644 --- a/crates/workspace-server/src/authority.rs +++ b/crates/workspace-server/src/authority.rs @@ -238,13 +238,6 @@ impl SqliteWorkspaceAuthority { }) } - // Contention fixtures must hold the candidate query's actual connection, - // not the API Store used only by some enrichment paths. - #[cfg(test)] - pub(crate) fn candidate_query_store_for_test(&self) -> SqliteWorkspaceStore { - self.store.clone() - } - /// Assignment self-report enrichment is a separate projection so retained /// responsibility and existing Ticket constructors stay distinct from status. pub fn ticket_assignment_tracking( @@ -3356,7 +3349,7 @@ mod tests { async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -3431,7 +3424,7 @@ mod tests { async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -3564,7 +3557,7 @@ mod tests { write_ticket(dir.path(), "00000000001J5", "Second ticket", "queued"); write_ticket(dir.path(), "00000000001J6", "Third ticket", "planning"); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), @@ -3996,7 +3989,7 @@ VALUES ('workspace-test', 'ticket', 4); async fn objective_mutations_write_sqlite_records_and_audit_events() { let dir = tempfile::tempdir().unwrap(); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), diff --git a/crates/workspace-server/src/config_source.rs b/crates/workspace-server/src/config_source.rs index 7ac608622..58158f464 100644 --- a/crates/workspace-server/src/config_source.rs +++ b/crates/workspace-server/src/config_source.rs @@ -795,7 +795,7 @@ mod tests { } async fn open_store() -> SqliteWorkspaceStore { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); store } @@ -1454,7 +1454,7 @@ mod tests { #[tokio::test] async fn invalid_candidate_is_never_persisted() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let main = current.snapshot.get(&path(MAIN_CONFIG_ENTRYPOINT)).unwrap(); @@ -1478,7 +1478,7 @@ mod tests { #[tokio::test] async fn valid_candidate_commits_snapshot_content_and_provenance_atomically() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let committed = store @@ -1496,7 +1496,7 @@ mod tests { #[tokio::test] async fn stale_cas_cannot_overwrite_newer_tree() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let request = commit_request(¤t, vec![update_main(¤t, "{ answer = 42; }")]); @@ -1514,7 +1514,7 @@ mod tests { #[tokio::test] async fn committed_content_remains_retrievable_after_later_commit() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store.upsert_workspace(&workspace()).await.unwrap(); let current = store.load_workspace_config("w-config").unwrap().unwrap(); let first = store @@ -1772,7 +1772,7 @@ mod tests { #[test] fn migration_creates_config_authority_tables() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store .with_conn(|conn| { for table in [ diff --git a/crates/workspace-server/src/db_executor.rs b/crates/workspace-server/src/db_executor.rs deleted file mode 100644 index 1c44ef8c9..000000000 --- a/crates/workspace-server/src/db_executor.rs +++ /dev/null @@ -1,957 +0,0 @@ -//! Bounded execution of whole synchronous DB operations, independent of Tokio. -//! -//! Accepted jobs belong to the workers, not their async waiters. Cancelling a -//! waiter does not cancel or retry SQL, release capacity, or release its lease. -use std::{ - collections::BTreeSet, - future::Future, - num::NonZeroUsize, - panic::{AssertUnwindSafe, catch_unwind}, - sync::{Arc, Mutex, OnceLock, mpsc}, -}; -use tokio::sync::{Notify, oneshot}; - -use crate::{Error, Result}; - -/// The default is one FIFO worker and at most 32 waiting operations. -#[derive(Clone, Copy, Debug)] -pub struct DbExecutorLimits { - pub workers: NonZeroUsize, - pub queued: NonZeroUsize, -} - -impl Default for DbExecutorLimits { - fn default() -> Self { - Self { - workers: NonZeroUsize::new(1).unwrap(), - queued: NonZeroUsize::new(32).unwrap(), - } - } -} - -/// A request can retain its lifecycle authority until actual SQL finishes. -/// Retention happens at submission, while the caller's task-local scope exists. -pub(crate) trait DbOperationLifetime: Send + Sync { - fn retain(&self) -> Box; -} - -tokio::task_local! { - static OPERATION_LIFETIME: Arc; -} - -pub(crate) async fn with_operation_lifetime( - lifetime: Arc, - future: F, -) -> F::Output { - OPERATION_LIFETIME.scope(lifetime, future).await -} - -#[derive(Default)] -struct Accepted { - last_job_id: u64, - unfinished: BTreeSet, -} - -#[derive(Default)] -struct Tracking { - accepted: Mutex, - finished: Notify, -} - -struct Completion { - tracking: Arc, - id: u64, -} - -impl Drop for Completion { - fn drop(&mut self) { - self.tracking - .accepted - .lock() - .unwrap() - .unfinished - .remove(&self.id); - self.tracking.finished.notify_waiters(); - } -} - -struct Job { - operation: Box, - // Field drop order releases authority before completion becomes observable. - _lifetime: Option>, - lifetime_source: Option>, - _completion: Completion, - runtime: Option, - diagnostics: tracing::Dispatch, - span: tracing::Span, -} - -fn report_unknown(accepted_job_id: u64, phase: &'static str) { - tracing::error!( - code = "db_operation_outcome_unknown", - accepted_job_id, - phase, - error = %Error::DbOperationOutcomeUnknown, - "DB job panicked; outcome unknown, operation will not be retried" - ); -} - -/// Clones share the queue and actual-job budget. Workers retain only the -/// receiver, so dropping the last sender closes the queue; accepted jobs finish -/// before the dedicated threads exit. No thread is created per store open. -#[derive(Clone)] -pub struct DbExecutor { - sender: Option>, - tracking: Arc, - max_jobs: usize, -} - -impl DbExecutor { - pub fn new(limits: DbExecutorLimits) -> Result { - let max_jobs = limits - .workers - .get() - .checked_add(limits.queued.get()) - .ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?; - let (sender, receiver) = mpsc::sync_channel::(limits.queued.get()); - let receiver = Arc::new(Mutex::new(receiver)); - for index in 0..limits.workers.get() { - let receiver = receiver.clone(); - std::thread::Builder::new() - .name(format!("workspace-db-{index}")) - .spawn(move || { - loop { - let next = receiver.lock().unwrap().recv(); - let Ok(job) = next else { break }; - let Job { - operation, - _lifetime: lifetime, - lifetime_source, - _completion: completion, - runtime, - diagnostics, - span, - } = job; - // Keep diagnostics with their submitting subscriber as - // well as the runtime; no process-global override. - let _diagnostics = tracing::dispatcher::set_default(&diagnostics); - let _span = span.enter(); - // Preserve the submitting runtime for synchronous Host - // post-commit hooks, including reply/lease destructors. - // enter() supplies context only: never drive a future - // or await/nest an executor submission in a transaction. - let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter); - // A cancelled waiter's discarded result may itself have - // a destructor. Keep both the lease and actual-job slot - // until that cleanup finishes, even if it panics. - let execute = || { - if catch_unwind(AssertUnwindSafe(operation)).is_err() { - report_unknown(completion.id, "reply_cleanup"); - } - if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() { - report_unknown(completion.id, "lifetime_cleanup"); - } - }; - // Synchronous post-commit continuations can admit another - // bounded job. Transfer the original lifecycle source as - // well, so its authority survives the parent job/waiter. - // This does not inject context into detached Tokio tasks. - if let Some(source) = lifetime_source { - OPERATION_LIFETIME.sync_scope(source, execute); - } else { - execute(); - } - drop(completion); - } - }) - .map_err(|_| Error::DbExecutorUnavailable)?; - } - Ok(Self { - sender: Some(sender), - tracking: Arc::default(), - max_jobs, - }) - } - - pub fn shared() -> Self { - static SHARED: OnceLock = OnceLock::new(); - SHARED - .get_or_init(|| { - Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self { - sender: None, - tracking: Arc::default(), - max_jobs: 0, - }) - }) - .clone() - } - - /// Submit without waiting for queue capacity. The closure must contain the - /// entire synchronous operation/transaction, with no nested async runtime. - /// The submitting Tokio handle, when present, is entered for synchronous - /// post-commit hooks and worker cleanup; it is never used to block_on SQL. - pub async fn run( - &self, - operation: impl FnOnce() -> std::result::Result + Send + 'static, - ) -> std::result::Result - where - T: Send + 'static, - E: From + Send + 'static, - { - self.submit(operation)? - .await - .unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into())) - } - - /// Admit a fire-and-forget operation immediately, without spawning a Tokio - /// task or waiting for capacity. Err means the operation was not accepted; - /// Ok means the worker owns it through execution and cleanup, even after - /// the submitting task is cancelled. A panic emits a structured unknown- - /// outcome diagnostic, never an automatic retry. drain() tracks these jobs. - pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> { - self.submit(move || { - operation(); - Ok::<_, Error>(()) - }) - .map(drop) - } - - /// Shared synchronous admission: neither dropped receivers nor cancelled - /// callers remove accepted work from the queue or the actual-job budget. - fn submit( - &self, - operation: impl FnOnce() -> std::result::Result + Send + 'static, - ) -> std::result::Result>, E> - where - T: Send + 'static, - E: From + Send + 'static, - { - let sender = self - .sender - .as_ref() - .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; - let runtime = tokio::runtime::Handle::try_current().ok(); - let diagnostics = tracing::dispatcher::get_default(Clone::clone); - let span = tracing::Span::current(); - let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok(); - let lifetime = lifetime_source.as_ref().map(|source| source.retain()); - let (reply, result) = oneshot::channel(); - // Serialize admission and acceptance snapshots. Worker completion cannot - // race acceptance, and cancelled waiters never alter this actual-job cap. - { - let mut accepted = self.tracking.accepted.lock().unwrap(); - if accepted.unfinished.len() >= self.max_jobs { - return Err(Error::DbCapacityExceeded.into()); - } - let id = accepted - .last_job_id - .checked_add(1) - .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; - // Never reuse the identity of a rejected job: its destructor may run - // after another submission has acquired the admission lock. - accepted.last_job_id = id; - let job = Job { - operation: Box::new(move || { - let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| { - report_unknown(id, "operation"); - Err(Error::DbOperationOutcomeUnknown.into()) - }); - let _ = reply.send(outcome); - }), - _lifetime: lifetime, - lifetime_source, - runtime, - diagnostics, - span, - _completion: Completion { - tracking: self.tracking.clone(), - id, - }, - }; - match sender.try_send(job) { - Ok(()) => { - accepted.unfinished.insert(id); - } - Err(error) => { - // The rejected Job's completion also locks tracking; drop it only - // after releasing the admission lock. - drop(accepted); - return Err(match error { - mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded, - mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable, - } - .into()); - } - } - } - Ok(result) - } - - /// Wait for every job accepted before this barrier is first polled (not - /// later submissions). This bypasses queue capacity and survives cancelled - /// job waiters. It does not close the queue or stop new submissions. - pub async fn drain(&self) -> Result<()> { - if self.sender.is_none() { - return Err(Error::DbExecutorUnavailable); - } - let through = self.tracking.accepted.lock().unwrap().last_job_id; - loop { - let notified = self.tracking.finished.notified(); - tokio::pin!(notified); - notified.as_mut().enable(); - if self - .tracking - .accepted - .lock() - .unwrap() - .unfinished - .first() - .is_none_or(|id| *id > through) - { - return Ok(()); - } - notified.await; - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rusqlite::{Connection, TransactionBehavior, ffi}; - use std::{ffi::c_void, time::Duration}; - - fn executor(queued: usize) -> DbExecutor { - DbExecutor::new(DbExecutorLimits { - workers: NonZeroUsize::new(1).unwrap(), - queued: NonZeroUsize::new(queued).unwrap(), - }) - .unwrap() - } - - fn database() -> (tempfile::TempDir, Connection, Connection) { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("jobs.sqlite"); - let owner = Connection::open(&path).unwrap(); - owner - .execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)") - .unwrap(); - let worker = Connection::open(&path).unwrap(); - (dir, owner, worker) - } - - struct BusyGate { - waiting: Option>, - release: mpsc::Receiver<()>, - } - - // A fixture-owned callback proves SQLite actually reached its lock wait. - // Unlike busy_handler's fn pointer API, the SQLite API carries a per-call - // context, so tests need neither process-global state nor guessed sleeps. - struct BusyRegistration { - db: *mut ffi::sqlite3, - _gate: Box, - } - - impl BusyRegistration { - fn install(conn: &Connection, gate: BusyGate) -> Self { - unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 { - // The Box remains registered only while this connection and - // BusyRegistration are alive, on this same dedicated worker. - let gate = unsafe { &mut *context.cast::() }; - if let Some(waiting) = gate.waiting.take() { - let _ = waiting.send(()); - if gate.release.recv().is_ok() { - return 1; - } - } - 0 - } - let mut gate = Box::new(gate); - // SAFETY: conn owns the handle; registration is dropped before conn. - let db = unsafe { conn.handle() }; - let result = unsafe { - ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast()) - }; - assert_eq!(result, ffi::SQLITE_OK); - Self { db, _gate: gate } - } - } - - impl Drop for BusyRegistration { - fn drop(&mut self) { - // SAFETY: unregister before releasing the callback's boxed context. - unsafe { - ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut()); - } - } - } - - async fn observed(future: impl Future) -> T { - tokio::time::timeout(Duration::from_secs(5), future) - .await - .expect("DB job did not reach the expected observable state") - } - - #[tokio::test(flavor = "current_thread")] - async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() { - let (_dir, mut owner, mut worker) = database(); - let lock = owner - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - let executor = executor(1); - let (waiting, blocked) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - let mut write = Box::pin(executor.run(move || -> Result<()> { - let _busy = BusyRegistration::install( - &worker, - BusyGate { - waiting: Some(waiting), - release: permit, - }, - ); - let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute("INSERT INTO writes VALUES (1)", [])?; - tx.commit()?; - Ok(()) - })); - assert!(futures::poll!(&mut write).is_pending()); - observed(blocked).await.unwrap(); - let control = tokio::spawn(async { 42 }); - assert_eq!(observed(control).await.unwrap(), 42); - // The timer itself is the contract under test, not synchronization. - observed(tokio::time::sleep(Duration::from_millis(1))).await; - assert!(futures::poll!(&mut write).is_pending()); - lock.rollback().unwrap(); - release.send(()).unwrap(); - observed(write).await.unwrap(); - executor.drain().await.unwrap(); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 1 - ); - } - - struct Lease(Arc); - struct Retained(Arc); - impl DbOperationLifetime for Lease { - fn retain(&self) -> Box { - self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Box::new(Retained(self.0.clone())) - } - } - impl Drop for Retained { - fn drop(&mut self) { - self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); - } - } - - #[tokio::test(flavor = "current_thread")] - async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() { - let (_dir, mut owner, mut worker) = database(); - let lock = owner - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - let executor = executor(1); - let (waiting, blocked) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - let mut running = Box::pin(executor.run(move || -> Result<()> { - let _busy = BusyRegistration::install( - &worker, - BusyGate { - waiting: Some(waiting), - release: permit, - }, - ); - let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute("INSERT INTO writes VALUES (1)", [])?; - tx.commit()?; - Ok(()) - })); - assert!(futures::poll!(&mut running).is_pending()); - observed(blocked).await.unwrap(); - let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let lease: Arc = Arc::new(Lease(count.clone())); - let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap(); - let mut queued = Box::pin(with_operation_lifetime( - lease, - executor.run(move || -> Result<()> { - let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - // FIFO is observable in durable SQL, not inferred from waiter order. - let first: i64 = - tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| { - row.get(0) - })?; - assert_eq!(first, 1); - tx.execute("INSERT INTO writes VALUES (2)", [])?; - tx.commit()?; - Ok(()) - }), - )); - assert!(futures::poll!(&mut queued).is_pending()); - drop(queued); - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); - assert!(matches!( - executor.run(|| Ok::<_, Error>(())).await, - Err(Error::DbCapacityExceeded) - )); - let mut drained = Box::pin(executor.drain()); - assert!(futures::poll!(&mut drained).is_pending()); - lock.rollback().unwrap(); - release.send(()).unwrap(); - observed(running).await.unwrap(); - observed(drained).await.unwrap(); - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 2 - ); - executor.run(|| Ok::<_, Error>(())).await.unwrap(); - } - - #[tokio::test(flavor = "current_thread")] - async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() { - let (dir, owner, mut parent_conn) = database(); - let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap(); - let parent = executor(1); - let child = executor(1); - let child_owner = child.clone(); - let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let source: Arc = Arc::new(Lease(count.clone())); - let (entered, running) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - let (child_entered, child_running) = oneshot::channel(); - let (finish_child, child_permit) = mpsc::channel(); - let mut operation = Box::pin(with_operation_lifetime( - source, - parent.run(move || -> Result<()> { - let _ = entered.send(()); - let _ = permit.recv(); - let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute("INSERT INTO writes VALUES (1)", [])?; - tx.commit()?; - child_owner.dispatch(move || { - let tx = child_conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - let committed: i64 = tx - .query_row("SELECT count(*) FROM writes", [], |row| row.get(0)) - .unwrap(); - assert_eq!(committed, 1, "child must see its parent's committed SQL"); - let _ = child_entered.send(()); - let _ = child_permit.recv(); - tx.execute("INSERT INTO writes VALUES (2)", []).unwrap(); - tx.commit().unwrap(); - })?; - Ok(()) - }), - )); - assert!(futures::poll!(&mut operation).is_pending()); - observed(running).await.unwrap(); - drop(operation); - release.send(()).unwrap(); - observed(child_running).await.unwrap(); - observed(parent.drain()).await.unwrap(); - assert_eq!( - count.load(std::sync::atomic::Ordering::SeqCst), - 1, - "accepted continuation lost authority when its parent finished" - ); - let mut drained = Box::pin(child.drain()); - assert!(futures::poll!(&mut drained).is_pending()); - finish_child.send(()).unwrap(); - observed(drained).await.unwrap(); - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 2 - ); - } - - #[tokio::test] - async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() { - let (_dir, owner, mut worker) = database(); - let executor = executor(1); - let error = executor - .run(move || -> Result<()> { - let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute("INSERT INTO writes VALUES (1)", [])?; - tx.execute("INSERT INTO writes VALUES (NULL)", [])?; - tx.commit()?; - Ok(()) - }) - .await - .unwrap_err(); - assert!(matches!(error, Error::Sqlite(_))); - executor.drain().await.unwrap(); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 0 - ); - executor.run(|| Ok::<_, Error>(())).await.unwrap(); - } - - #[tokio::test] - async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() { - let (_dir, owner, mut worker) = database(); - let executor = executor(1); - let error = executor - .run(move || -> Result<()> { - let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute("INSERT INTO writes VALUES (1)", [])?; - tx.commit()?; - panic!("lost outcome after commit"); - }) - .await - .unwrap_err(); - assert!(matches!(error, Error::DbOperationOutcomeUnknown)); - executor.drain().await.unwrap(); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 1 - ); - assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42); - } - - #[tokio::test] - async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() { - let (_dir, owner, mut worker) = database(); - let executor = executor(1); - let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let lease: Arc = Arc::new(Lease(count.clone())); - let error = with_operation_lifetime( - lease, - executor.run(move || -> Result<()> { - let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute("INSERT INTO writes VALUES (1)", [])?; - panic!("lost outcome before commit"); - }), - ) - .await - .unwrap_err(); - assert!(matches!(error, Error::DbOperationOutcomeUnknown)); - executor.drain().await.unwrap(); - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 0 - ); - executor.run(|| Ok::<_, Error>(())).await.unwrap(); - } - - struct DiscardedReplyHook { - expected_runtime: tokio::runtime::Id, - lease_count: Arc, - dropped: Option>, - } - - impl Drop for DiscardedReplyHook { - fn drop(&mut self) { - let runtime = - tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context"); - let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); - let same_runtime = runtime.id() == self.expected_runtime; - let dropped = self.dropped.take().unwrap(); - tokio::spawn(async move { - let _ = dropped.send((same_runtime, retained)); - }); - } - } - - #[tokio::test(flavor = "current_thread")] - async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop() - { - let (_dir, owner, mut worker) = database(); - let executor = executor(1); - let (started, running) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - let mut blocker = Box::pin(executor.run(move || -> Result<()> { - let _ = started.send(()); - let _ = permit.recv(); - Ok(()) - })); - assert!(futures::poll!(&mut blocker).is_pending()); - observed(running).await.unwrap(); - let expected_runtime = tokio::runtime::Handle::current().id(); - let (hook, published) = oneshot::channel(); - let (finish, finishing) = mpsc::channel(); - let (dropped, reply_dropped) = oneshot::channel(); - let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let lease: Arc = Arc::new(Lease(count.clone())); - let operation_count = count.clone(); - let mut queued = Box::pin(with_operation_lifetime( - lease, - executor.run(move || -> Result<_> { - let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute("INSERT INTO writes VALUES (1)", [])?; - tx.commit()?; - // Model synchronous Host hooks: publish only after commit, checking - // try_current and spawning without driving any future on this worker. - let runtime = tokio::runtime::Handle::try_current() - .expect("post-commit hook lost runtime context"); - assert_eq!(runtime.id(), expected_runtime); - tokio::spawn(async move { - let _ = hook.send(()); - }); - let _ = finishing.recv(); - Ok(DiscardedReplyHook { - expected_runtime, - lease_count: operation_count, - dropped: Some(dropped), - }) - }), - )); - assert!(futures::poll!(&mut queued).is_pending()); - drop(queued); - release.send(()).unwrap(); - observed(blocker).await.unwrap(); - observed(published).await.unwrap(); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 1 - ); - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); - let mut drain = Box::pin(executor.drain()); - assert!(futures::poll!(&mut drain).is_pending()); - finish.send(()).unwrap(); - assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1)); - observed(drain).await.unwrap(); - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); - executor.run(|| Ok::<_, Error>(())).await.unwrap(); - } - - #[tokio::test(flavor = "current_thread")] - async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() { - let executor = DbExecutor::new(DbExecutorLimits { - workers: NonZeroUsize::new(2).unwrap(), - queued: NonZeroUsize::new(1).unwrap(), - }) - .unwrap(); - let mut releases = Vec::new(); - // Observe each worker executing before submitting the next operation, - // so queue occupancy and overload are deterministic, not timing guesses. - for _ in 0..2 { - let (started, running) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - let mut operation = Box::pin(executor.run(move || -> Result<()> { - let _ = started.send(()); - let _ = permit.recv(); - Ok(()) - })); - assert!(futures::poll!(&mut operation).is_pending()); - observed(running).await.unwrap(); - drop(operation); - releases.push(release); - } - let (finished, mut queued_finished) = oneshot::channel(); - let mut queued = Box::pin(executor.run(move || -> Result<()> { - let _ = finished.send(()); - Ok(()) - })); - assert!(futures::poll!(&mut queued).is_pending()); - drop(queued); - let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - for _ in 0..8 { - let rejected_runs = rejected_runs.clone(); - assert!(matches!( - executor - .run(move || { - rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Ok::<_, Error>(()) - }) - .await, - Err(Error::DbCapacityExceeded) - )); - } - assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); - assert!(matches!( - queued_finished.try_recv(), - Err(oneshot::error::TryRecvError::Empty) - )); - let mut drain = Box::pin(executor.drain()); - assert!(futures::poll!(&mut drain).is_pending()); - for release in releases { - release.send(()).unwrap(); - } - observed(drain).await.unwrap(); - observed(queued_finished).await.unwrap(); - executor.run(|| Ok::<_, Error>(())).await.unwrap(); - } - - #[derive(Default)] - struct UnknownDiagnostic { - code: Option, - phase: Option, - accepted_job_id: Option, - } - - impl tracing::field::Visit for UnknownDiagnostic { - fn record_str(&mut self, field: &tracing::field::Field, value: &str) { - match field.name() { - "code" => self.code = Some(value.to_owned()), - "phase" => self.phase = Some(value.to_owned()), - _ => {} - } - } - fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { - if field.name() == "accepted_job_id" { - self.accepted_job_id = Some(value); - } - } - fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} - } - - #[derive(Clone, Default)] - struct DiagnosticCapture(Arc>>); - - impl tracing_subscriber::Layer for DiagnosticCapture { - fn on_event( - &self, - event: &tracing::Event<'_>, - _: tracing_subscriber::layer::Context<'_, S>, - ) { - let mut diagnostic = UnknownDiagnostic::default(); - event.record(&mut diagnostic); - self.0.lock().unwrap().push(diagnostic); - } - } - - struct DispatchCleanup { - expected_runtime: tokio::runtime::Id, - lease_count: Arc, - started: Option>, - release: mpsc::Receiver<()>, - } - - impl Drop for DispatchCleanup { - fn drop(&mut self) { - // No panicking assertions during unwind: report observations to the - // test, and gate cleanup so drain/lifetime behavior is observable. - let same_runtime = tokio::runtime::Handle::try_current() - .is_ok_and(|runtime| runtime.id() == self.expected_runtime); - let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); - if let Some(started) = self.started.take() { - let _ = started.send((same_runtime, retained)); - } - let _ = self.release.recv(); - } - } - - #[tokio::test(flavor = "current_thread")] - async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry() - { - use tracing_subscriber::layer::SubscriberExt; - let (_dir, owner, mut worker) = database(); - let executor = executor(1); - let (started, running) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - let mut blocker = Box::pin(executor.run(move || -> Result<()> { - let _ = started.send(()); - let _ = permit.recv(); - Ok(()) - })); - assert!(futures::poll!(&mut blocker).is_pending()); - observed(running).await.unwrap(); - drop(blocker); // Cancellation does not free its running slot. - let expected_runtime = tokio::runtime::Handle::current().id(); - let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let lease: Arc = Arc::new(Lease(count.clone())); - let operation_count = count.clone(); - let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let operation_attempts = attempts.clone(); - let (cleanup_started, cleaning_up) = oneshot::channel(); - let (finish, finishing) = mpsc::channel(); - // Model multiple tracing contexts without setting a global subscriber. - // This also prevents tracing's single-subscriber callsite optimization - // from caching an unsubscribed parallel test's Interest::never here. - let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry()); - let captured = DiagnosticCapture::default(); - let diagnostics = - tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone())); - with_operation_lifetime(lease, async { - tracing::dispatcher::with_default(&diagnostics, || { - executor.dispatch(move || { - let _cleanup = DispatchCleanup { - expected_runtime, - lease_count: operation_count, - started: Some(cleanup_started), - release: finishing, - }; - operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - let tx = worker - .transaction_with_behavior(TransactionBehavior::Immediate) - .unwrap(); - tx.execute("INSERT INTO writes VALUES (1)", []).unwrap(); - tx.commit().unwrap(); - panic!("dispatched post-commit outcome lost"); - }) - }) - }) - .await - .unwrap(); // Dispatch is admitted synchronously; no waiter survives. - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); - let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let rejected_operation = rejected_runs.clone(); - assert!(matches!( - executor.dispatch(move || { - rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - }), - Err(Error::DbCapacityExceeded) - )); - release.send(()).unwrap(); - assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1)); - assert_eq!( - owner - .query_row("SELECT count(*) FROM writes", [], |row| row - .get::<_, i64>(0)) - .unwrap(), - 1 - ); - let mut drain = Box::pin(executor.drain()); - assert!(futures::poll!(&mut drain).is_pending()); - finish.send(()).unwrap(); - observed(drain).await.unwrap(); - assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); - assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); - assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); - { - let events = captured.0.lock().unwrap(); - assert_eq!(events.len(), 1); - assert_eq!( - events[0].code.as_deref(), - Some("db_operation_outcome_unknown") - ); - assert_eq!(events[0].phase.as_deref(), Some("operation")); - assert!(events[0].accepted_job_id.is_some()); - } - let (completed, completion) = oneshot::channel(); - executor - .dispatch(move || { - let _ = completed.send(()); - }) - .unwrap(); - observed(executor.drain()).await.unwrap(); - observed(completion).await.unwrap(); - } -} diff --git a/crates/workspace-server/src/frozen_content_state_tests.rs b/crates/workspace-server/src/frozen_content_state_tests.rs index 954d42ccf..138b83683 100644 --- a/crates/workspace-server/src/frozen_content_state_tests.rs +++ b/crates/workspace-server/src/frozen_content_state_tests.rs @@ -443,7 +443,6 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store @@ -492,7 +491,6 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store diff --git a/crates/workspace-server/src/frozen_repository_key_tests.rs b/crates/workspace-server/src/frozen_repository_key_tests.rs index 798f1575d..adeef98fe 100644 --- a/crates/workspace-server/src/frozen_repository_key_tests.rs +++ b/crates/workspace-server/src/frozen_repository_key_tests.rs @@ -469,7 +469,6 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }; let migrated = store @@ -653,7 +652,6 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let archived = store @@ -1029,7 +1027,6 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = @@ -1182,7 +1179,6 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() { // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = diff --git a/crates/workspace-server/src/frozen_runtime_binding_tests.rs b/crates/workspace-server/src/frozen_runtime_binding_tests.rs index 9c8bef71f..fcf7dcc97 100644 --- a/crates/workspace-server/src/frozen_runtime_binding_tests.rs +++ b/crates/workspace-server/src/frozen_runtime_binding_tests.rs @@ -533,7 +533,6 @@ mod existing_workspace_provisioning { // This fixture contains only the schema-87 cutover authorities, not the // unrelated full Server schema. Exercise the real service/store methods. Arc::new(SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }) } @@ -645,7 +644,6 @@ mod existing_workspace_provisioning { drop(service); drop(store); let store = Arc::new(SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())), }); let service = WorkspaceSigningIdentityService::new( @@ -818,7 +816,7 @@ mod current_authority { use super::*; fn binding_store() -> (SqliteWorkspaceStore, WorkspaceRuntimeBinding) { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-key").unwrap(); store.with_conn(|conn| { diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 0b41a9a47..c0fe34c99 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -23,10 +23,7 @@ use std::{ }; use workdir::{ Workdir, WorkdirError, WorkdirSessionHandle, - http::{ - OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization, - WorkdirHttpAuthorizationFuture, - }, + http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization}, }; use worker_runtime::RuntimeWorkspaceScope; use worker_runtime::catalog::{ @@ -3480,38 +3477,6 @@ impl WorkspaceRuntimeAuthorization { } } - /// Offload the complete live-binding check and request signing operation. - /// Claims are minted inside the bounded job, not cached before admission. - pub(crate) async fn issue_async( - &self, - method: String, - path_and_query: String, - operation: String, - worker_id: Option, - body: Vec, - ) -> Result { - let executor = self.store.db_executor(); - let authorization = self.clone(); - executor - .run(move || { - Ok::<_, Error>(authorization.issue( - &method, - &path_and_query, - &operation, - worker_id.as_deref(), - &body, - )) - }) - .await - .map_err(|error| { - diagnostic( - "workspace_runtime_authorization_unavailable", - HostDiagnosticSeverity::Error, - error.to_string(), - ) - })? - } - pub(crate) fn issue( &self, method: &str, @@ -3684,24 +3649,25 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization { impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization { fn bearer_token( &self, - method: String, - path_and_query: String, - body: Vec, - ) -> WorkdirHttpAuthorizationFuture { - let authorization = self.clone(); - Box::pin(async move { - if let Some(authorizer) = &authorization.workspace_authorization { - let operation = workspace_runtime_operation(&method, &path_and_query).to_string(); - return authorizer - .issue_async(method, path_and_query, operation, None, body) - .await - .map_err(|error| WorkdirError::Unavailable(error.message)); - } - authorization.fallback_bearer_token.ok_or_else(|| { - WorkdirError::Unavailable( - "remote Runtime does not have bearer authorization configured".to_string(), + method: &str, + path_and_query: &str, + body: &[u8], + ) -> Result { + if let Some(authorization) = &self.workspace_authorization { + return authorization + .issue( + method, + path_and_query, + workspace_runtime_operation(method, path_and_query), + None, + body, ) - }) + .map_err(|error| WorkdirError::Unavailable(error.message)); + } + self.fallback_bearer_token.clone().ok_or_else(|| { + WorkdirError::Unavailable( + "remote Runtime does not have bearer authorization configured".to_string(), + ) }) } } @@ -6382,79 +6348,6 @@ mod tests { use std::sync::{Arc, Mutex}; use std::thread; - #[tokio::test] - async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() { - use crate::db_executor::{DbExecutor, DbExecutorLimits}; - use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding}; - use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore; - - let executor = DbExecutor::new(DbExecutorLimits { - workers: std::num::NonZeroUsize::new(1).unwrap(), - queued: std::num::NonZeroUsize::new(1).unwrap(), - }) - .unwrap(); - let store = Arc::new( - SqliteWorkspaceStore::in_memory_for_test() - .unwrap() - .with_db_executor(executor.clone()), - ); - let service = WorkspaceSigningIdentityService::new( - store.clone(), - Arc::new(InMemoryWorkspaceSigningMaterialStore::default()), - ); - let authorization = RemoteWorkdirAuthorization { - workspace_authorization: Some(WorkspaceRuntimeAuthorization::new( - store, - service, - "https://backend.test", - WorkspaceRuntimeBinding { - workspace_id: "workspace".into(), - runtime_id: "runtime".into(), - display_name: "Runtime".into(), - base_url: "https://runtime.test".into(), - public_key_fingerprint: "unused".into(), - public_key: "unused".into(), - binding_id: "binding".into(), - authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, - created_at: "1".into(), - updated_at: "1".into(), - revoked_at: None, - }, - )), - fallback_bearer_token: Some("must-not-be-used".into()), - }; - - // Hold the running slot and fill the only queue slot. The callback must - // reject admission, not run inline or silently switch authentication. - let (release, wait) = std::sync::mpsc::channel::<()>(); - let (started, running) = tokio::sync::oneshot::channel(); - executor - .dispatch(move || { - let _ = started.send(()); - let _ = wait.recv_timeout(Duration::from_secs(5)); - }) - .unwrap(); - tokio::time::timeout(Duration::from_secs(5), running) - .await - .unwrap() - .unwrap(); - executor.dispatch(|| {}).unwrap(); - let error = authorization - .bearer_token( - "POST".into(), - "/v1/workdir-sessions/s/operations".into(), - b"{}".to_vec(), - ) - .await - .unwrap_err(); - assert!(matches!( - error, - WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string() - )); - drop(release); - executor.drain().await.unwrap(); - } - #[test] fn ticket_worker_spawn_intent_uses_worker_wire_role() { let intent = WorkerSpawnIntent::TicketRole { diff --git a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs index 29c0cb52c..f68964f35 100644 --- a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs +++ b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs @@ -32,8 +32,7 @@ struct Fixture { impl Fixture { async fn new() -> Self { let temp = tempfile::tempdir().unwrap(); - let store = - Arc::new(SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap()); store .upsert_account(&AccountRecord { account_id: "owner".into(), diff --git a/crates/workspace-server/src/legacy_evidence_column_tests.rs b/crates/workspace-server/src/legacy_evidence_column_tests.rs index 33994b292..a59f2a0ca 100644 --- a/crates/workspace-server/src/legacy_evidence_column_tests.rs +++ b/crates/workspace-server/src/legacy_evidence_column_tests.rs @@ -45,7 +45,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() { let artifacts = evidence_rows(&conn, "artifacts"); let receipts = evidence_rows(&conn, "repository_secret_legacy_receipts"); drop(conn); - let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); upgraded .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -73,7 +73,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() { Ok(()) }) .unwrap(); - for store in [upgraded, SqliteWorkspaceStore::in_memory_for_test().unwrap()] { + for store in [upgraded, SqliteWorkspaceStore::in_memory().unwrap()] { store.with_conn(|conn| { let mut stmt = conn.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")?; let objects = stmt.query_map([],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?)))?.collect::>>()?; diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index eb113b079..fefe24d92 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -10,7 +10,6 @@ pub mod backend_job; pub mod companion; pub mod config; pub mod config_source; -pub mod db_executor; pub mod feature_storage; pub mod hosts; pub mod memory_backend; @@ -160,12 +159,6 @@ pub enum Error { WorkspaceSigningIdentity { code: String, message: String }, #[error("Workspace Drive error: {0}")] Drive(#[from] workspace_drive::Error), - #[error("DB execution capacity exceeded")] - DbCapacityExceeded, - #[error("DB executor unavailable")] - DbExecutorUnavailable, - #[error("DB operation outcome unknown; do not automatically retry")] - DbOperationOutcomeUnknown, #[error("store error: {0}")] Store(String), } diff --git a/crates/workspace-server/src/memory_staging.rs b/crates/workspace-server/src/memory_staging.rs index 631d1b33e..ff8caadf7 100644 --- a/crates/workspace-server/src/memory_staging.rs +++ b/crates/workspace-server/src/memory_staging.rs @@ -217,7 +217,7 @@ mod tests { async fn authority() -> (TempDir, SqliteWorkspaceAuthority) { let temp = TempDir::new().unwrap(); let db_path = temp.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-test".to_string(), diff --git a/crates/workspace-server/src/migration_entrypoint_tests.rs b/crates/workspace-server/src/migration_entrypoint_tests.rs index 41329a730..d0eb2375d 100644 --- a/crates/workspace-server/src/migration_entrypoint_tests.rs +++ b/crates/workspace-server/src/migration_entrypoint_tests.rs @@ -32,7 +32,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() { drop(conn); for attempt in 0..3 { let error = match attempt { - 0 => SqliteWorkspaceStore::open_for_test(&path).err().unwrap(), + 0 => SqliteWorkspaceStore::open(&path).err().unwrap(), 1 => SqliteWorkspaceStore::migration_plan(&path).unwrap_err(), _ => SqliteWorkspaceStore::migrate_database(&path).unwrap_err(), }; @@ -62,7 +62,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() { ) .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); assert_eq!(conn.query_row("SELECT input_fingerprint FROM worker_removal_operations WHERE operation_id='removal'",[],|r|r.get::<_,String>(0))?,"legacy-receipt"); @@ -109,7 +109,7 @@ fn migration_dry_run_and_apply_share_the_original_repository_secret_authority() assert!(!table_exists(&conn, "server_secret_objects").unwrap()); drop(conn); assert_eq!(SqliteWorkspaceStore::migrate_database(&path).unwrap(), plan); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?,LATEST_SCHEMA_VERSION); let (nonce, mut sealed): (Vec,Vec) = conn.query_row( diff --git a/crates/workspace-server/src/repository_access.rs b/crates/workspace-server/src/repository_access.rs index c8a13c3f4..304909848 100644 --- a/crates/workspace-server/src/repository_access.rs +++ b/crates/workspace-server/src/repository_access.rs @@ -2290,7 +2290,7 @@ mod tests { ) { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); let timestamp = now(); for workspace_id in ["workspace-a", "workspace-b"] { futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord { diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index 4aa310885..aa6ca692d 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -1116,7 +1116,7 @@ mod tests { WorkerId::from_legacy_u64(1) } fn setup() -> SqliteWorkspaceStore { - let s = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let s = SqliteWorkspaceStore::in_memory().unwrap(); s.with_conn(|c| { c.execute( "INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) \ diff --git a/crates/workspace-server/src/runtime_subscription.rs b/crates/workspace-server/src/runtime_subscription.rs index 998786c38..3a7fdd0bd 100644 --- a/crates/workspace-server/src/runtime_subscription.rs +++ b/crates/workspace-server/src/runtime_subscription.rs @@ -965,7 +965,7 @@ async fn connect_runtime( let mut request = endpoint .into_client_request() .map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?; - if let Some(token) = runtime_token(config, workspace_id).await? { + if let Some(token) = runtime_token(config, workspace_id)? { request.headers_mut().insert( "authorization", format!("Bearer {token}") @@ -1021,20 +1021,13 @@ fn runtime_endpoint(base_url: &str) -> String { format!("{base}/v1/protocol/ws") } } -async fn runtime_token( +fn runtime_token( config: &RemoteRuntimeConfig, _workspace_id: &str, ) -> Result, String> { if let Some(authorization) = config.workspace_authorization.as_ref() { return authorization - .issue_async( - "GET".to_string(), - "/v1/protocol/ws".to_string(), - "workers:list".to_string(), - None, - Vec::new(), - ) - .await + .issue("GET", "/v1/protocol/ws", "workers:list", None, &[]) .map(Some) .map_err(|error| error.message); } diff --git a/crates/workspace-server/src/schema_equivalence_tests.rs b/crates/workspace-server/src/schema_equivalence_tests.rs index 4063a675b..37f5e218a 100644 --- a/crates/workspace-server/src/schema_equivalence_tests.rs +++ b/crates/workspace-server/src/schema_equivalence_tests.rs @@ -38,8 +38,8 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("upgraded.db"); prepare_retained_schema(&path, 85); - let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory().unwrap(); let mut mismatches = Vec::new(); for table in [ "workspace_tags", diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index a37c91975..5ed22bcc8 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -885,7 +885,7 @@ impl ExternalProviderWorkdirSession { } } - async fn audit_operation( + fn audit_operation( &self, operation_id: &ExternalWorkdirOperationId, operation_kind: &str, @@ -894,34 +894,25 @@ impl ExternalProviderWorkdirSession { let Some(audit) = &self.audit else { return Ok(()); }; - let audit = audit.clone(); - let operation_id = operation_id.clone(); - let operation_kind = operation_kind.to_owned(); - let outcome = outcome.to_owned(); - let connection = self.connection.clone(); audit .store - .db_executor() - .run(move || { - audit.store.record_external_workdir_audit( - &audit.workspace_id, - "worker", - &audit.actor_key, - "external_workdir_operation", - &connection.grant_id, - Some(operation_id.as_str()), - &outcome, - &format!( - "workdir_id={} attachment_alias={} operation_kind={} generation={}", - connection.workdir_id, - audit.attachment_alias, - operation_kind, - connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - }) - .await + .record_external_workdir_audit( + &audit.workspace_id, + "worker", + &audit.actor_key, + "external_workdir_operation", + &self.connection.grant_id, + Some(operation_id.as_str()), + outcome, + &format!( + "workdir_id={} attachment_alias={} operation_kind={} generation={}", + self.connection.workdir_id, + audit.attachment_alias, + operation_kind, + self.connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) .map_err(|error| { workdir::WorkdirError::Unavailable(format!( "External Workdir operation audit failed: {error}" @@ -969,8 +960,7 @@ impl ExternalProviderWorkdirSession { } let operation_id = ExternalWorkdirOperationId::new(format!("op-{}", Uuid::now_v7())) .map_err(workdir::WorkdirError::InvalidArgument)?; - self.audit_operation(&operation_id, operation_kind, "requested") - .await?; + self.audit_operation(&operation_id, operation_kind, "requested")?; let (response, receive) = tokio::sync::oneshot::channel(); let mut cancel_guard = ExternalOperationCancelGuard { sender: self.connection.sender.clone(), @@ -987,8 +977,7 @@ impl ExternalProviderWorkdirSession { .await .is_err() { - self.audit_operation(&operation_id, operation_kind, "provider_unavailable") - .await?; + self.audit_operation(&operation_id, operation_kind, "provider_unavailable")?; return Err(workdir::WorkdirError::Unavailable( "External Workdir provider disconnected".to_string(), )); @@ -1018,8 +1007,7 @@ impl ExternalProviderWorkdirSession { } else { "failed" }, - ) - .await?; + )?; result } @@ -1589,7 +1577,6 @@ fn orchestrator_attention_state( #[derive(Clone)] pub struct WorkspaceApi { - host_executors: HostExecutors, pub(crate) config: ServerConfig, pub(crate) store: Arc, config_store: Arc, @@ -1688,7 +1675,6 @@ impl ServerApiContractService { struct WorkerRemovalService { workspace_id: String, store: Arc, - host_executors: HostExecutors, runtime: Weak, workdir_sessions: Arc>, workdir_session_locks: Arc>>>>, @@ -1702,7 +1688,6 @@ impl WorkerRemovalService { Self { workspace_id: api.config.workspace_id.clone(), store: api.store.clone(), - host_executors: api.host_executors.clone(), runtime: Arc::downgrade(&api.runtime), workdir_sessions: api.workdir_sessions.clone(), workdir_session_locks: api.workdir_session_locks.clone(), @@ -1868,30 +1853,6 @@ impl WorkerRemovalService { target_runtime_id: &str, target_worker_id: &str, reason: &str, - ) -> std::result::Result { - let service = self.clone(); - let runtime_id = target_runtime_id.to_owned(); - let worker_id = target_worker_id.to_owned(); - let reason = reason.to_owned(); - run_host_workflow( - self.host_executors.host().map_err(|e| e.to_string())?, - async move { - service - .execute_async_on_host(source, &runtime_id, &worker_id, &reason) - .await - .map_err(Error::Store) - }, - ) - .await - .map_err(|error| error.to_string()) - } - - async fn execute_async_on_host( - &self, - source: crate::worker_source::VerifiedWorkerMutationSource, - target_runtime_id: &str, - target_worker_id: &str, - reason: &str, ) -> std::result::Result { let reason = reason.trim(); if reason.is_empty() || reason.len() > 512 { @@ -1963,39 +1924,11 @@ impl WorkerRemovalService { )); } - self.execute_target_removal_on_host(&runtime, &target, reason, None) + self.execute_target_removal(&runtime, &target, reason, None) .await } async fn execute_target_removal( - &self, - _runtime: &RuntimeRegistry, - target: &RuntimeWorkerRef, - reason: &str, - parent_workspace_deletion_operation_id: Option<&str>, - ) -> std::result::Result { - let runtime = self - .runtime - .upgrade() - .ok_or_else(|| "Workspace Runtime registry is unavailable".to_string())?; - let service = self.clone(); - let target = target.clone(); - let reason = reason.to_owned(); - let parent = parent_workspace_deletion_operation_id.map(str::to_owned); - run_host_workflow( - self.host_executors.host().map_err(|e| e.to_string())?, - async move { - service - .execute_target_removal_on_host(&runtime, &target, &reason, parent.as_deref()) - .await - .map_err(Error::Store) - }, - ) - .await - .map_err(|error| error.to_string()) - } - - async fn execute_target_removal_on_host( &self, runtime: &RuntimeRegistry, target: &RuntimeWorkerRef, @@ -2244,7 +2177,6 @@ use workspace_admission::WorkspaceAdmission; #[derive(Clone)] pub struct WorkspaceServerApi { - host_executors: HostExecutors, template: Arc, store: Arc, catalog: WorkspaceCatalogService, @@ -2275,7 +2207,6 @@ impl WorkspaceServerApi { workspace_signing_material_root(&template.database_path), )); Self { - host_executors: HostExecutors::shared(), template: Arc::new(template), catalog: WorkspaceCatalogService::new(store.clone(), signing_materials.clone()), signing_materials, @@ -2291,23 +2222,6 @@ impl WorkspaceServerApi { } } - #[cfg(test)] - fn new_for_test(template: ServerConfig, store: Arc) -> Self { - let mut api = Self::new(template, store); - api.host_executors = HostExecutors::default(); - api - } - - // Own the complete synchronous store stage; callers retain their async - // lifecycle guards across it and never split a transaction around await. - async fn execute_store_stage( - &self, - operation: impl FnOnce(Arc) -> Result + Send + 'static, - ) -> Result { - let store = self.store.clone(); - self.store.db_executor().run(move || operation(store)).await - } - async fn admission_for_workspace(&self, workspace_id: &str) -> Arc { workspace_admission_gate(&self.admission_gates, workspace_id).await } @@ -2320,22 +2234,11 @@ impl WorkspaceServerApi { let Some(workspace) = self.store.get_workspace(workspace_id).await? else { return Ok(None); }; - let store = self.store.clone(); - let id = workspace_id.to_owned(); - let repositories = self - .store - .db_executor() - .run(move || store.list_repositories(&id)) - .await?; + let repositories = self.store.list_repositories(workspace_id)?; let config = self .template .for_catalog_workspace(&workspace, repositories)?; - let api = WorkspaceApi::new_with_host_executors( - config, - self.store.clone(), - self.host_executors.clone(), - ) - .await?; + let api = WorkspaceApi::new(config, self.store.clone()).await?; apis.insert(workspace_id.to_string(), api.clone()); Ok(Some(api)) } @@ -2348,46 +2251,34 @@ impl WorkspaceServerApi { drop(running); let api = self.clone(); tokio::spawn(async move { - if let Err(error) = api.execute_workspace_deletion(&operation_id).await { - tracing::warn!(%error, %operation_id, "Workspace deletion execution did not complete"); - if !matches!(error, Error::DbOperationOutcomeUnknown) { - let id = operation_id.clone(); - let _ = api - .execute_store_stage(move |store| { - let operation = store - .workspace_deletion_operation_for_recovery(&id) - .ok() - .flatten(); - let child_operation_ids = operation - .as_ref() - .map(|operation| operation.child_operation_ids.as_slice()) - .unwrap_or_default(); - let blockers = operation - .as_ref() - .map(|operation| operation.blockers.as_slice()) - .unwrap_or_default(); - store - .update_workspace_deletion_operation( - &id, - WorkspaceDeletionState::Failed, - child_operation_ids, - blockers, - Some("workspace_deletion_execution_failed"), - ) - .map(drop) - }) - .await; - } + if api.execute_workspace_deletion(&operation_id).await.is_err() { + let operation = api + .store + .workspace_deletion_operation_for_recovery(&operation_id) + .ok() + .flatten(); + let child_operation_ids = operation + .as_ref() + .map(|operation| operation.child_operation_ids.as_slice()) + .unwrap_or_default(); + let blockers = operation + .as_ref() + .map(|operation| operation.blockers.as_slice()) + .unwrap_or_default(); + let _ = api.store.update_workspace_deletion_operation( + &operation_id, + WorkspaceDeletionState::Failed, + child_operation_ids, + blockers, + Some("workspace_deletion_execution_failed"), + ); } api.running_deletions.lock().await.remove(&operation_id); }); } async fn recover_workspace_deletions(&self) -> Result<()> { - let operations = self - .execute_store_stage(|store| store.resumable_workspace_deletion_operation_ids()) - .await?; - for operation_id in operations { + for operation_id in self.store.resumable_workspace_deletion_operation_ids()? { self.schedule_workspace_deletion(operation_id).await; } Ok(()) @@ -2398,20 +2289,16 @@ impl WorkspaceServerApi { actor_account_id: &str, workspace_id: &str, ) -> Result { - let actor = actor_account_id.to_owned(); - let id = workspace_id.to_owned(); let mut preflight = self - .execute_store_stage(move |store| store.workspace_deletion_preflight(&actor, &id)) - .await?; + .store + .workspace_deletion_preflight(actor_account_id, workspace_id)?; let Some(api) = self.api_for_workspace(workspace_id).await? else { return Err(Error::InvalidInput("Workspace does not exist".to_string())); }; - let id = workspace_id.to_owned(); - let workers = self - .execute_store_stage(move |store| store.list_worker_registry(&id, i64::MAX as usize)) - .await?; - api.host_executors.host()?.run(move || -> Result<_> { - for registry_worker in workers { + for registry_worker in self + .store + .list_worker_registry(workspace_id, i64::MAX as usize)? + { let worker_key = registry_worker.display_name; match api.runtime.worker(®istry_worker.worker) { Ok(worker) if worker.state == "stopped" => {} @@ -2434,30 +2321,23 @@ impl WorkspaceServerApi { crate::workspace_deletion::bound_workspace_deletion_blockers(&mut preflight.blockers); preflight.can_delete = preflight.blockers.is_empty(); Ok(preflight) - }).await } async fn execute_workspace_deletion( &self, operation_id: &str, ) -> Result { - let id = operation_id.to_owned(); - let operation = self - .execute_store_stage(move |store| { - let prior = store - .workspace_deletion_operation_for_recovery(&id)? - .ok_or_else(|| { - Error::InvalidInput("Workspace deletion operation".to_string()) - })?; - store.update_workspace_deletion_operation( - &id, - WorkspaceDeletionState::Running, - &prior.child_operation_ids, - &[], - None, - ) - }) - .await?; + let prior = self + .store + .workspace_deletion_operation_for_recovery(operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; + let operation = self.store.update_workspace_deletion_operation( + operation_id, + WorkspaceDeletionState::Running, + &prior.child_operation_ids, + &[], + None, + )?; let api = self .api_for_workspace(&operation.workspace_id) .await? @@ -2466,13 +2346,10 @@ impl WorkspaceServerApi { api.fence_drive().await?; let mut child_operation_ids = operation.child_operation_ids.clone(); let mut blockers = Vec::new(); - let workspace_id = operation.workspace_id.clone(); - let workers = self - .execute_store_stage(move |store| { - store.list_worker_registry(&workspace_id, i64::MAX as usize) - }) - .await?; - for worker in workers { + for worker in self + .store + .list_worker_registry(&operation.workspace_id, i64::MAX as usize)? + { let worker_key = worker.display_name.clone(); let target = worker.worker; let response = WorkerRemovalService::new(&api) @@ -2484,17 +2361,13 @@ impl WorkspaceServerApi { ) .await .map_err(Error::Store)?; - let id = operation_id.to_owned(); child_operation_ids = self - .execute_store_stage(move |store| { - store - .workspace_deletion_operation_for_recovery(&id)? - .ok_or_else(|| { - Error::Store("Workspace deletion operation disappeared".to_string()) - }) - .map(|operation| operation.child_operation_ids) - }) - .await?; + .store + .workspace_deletion_operation_for_recovery(operation_id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + })? + .child_operation_ids; if response.status != 200 { blockers.push(WorkspaceDeletionBlocker { kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, @@ -2506,36 +2379,22 @@ impl WorkspaceServerApi { } if blockers.is_empty() { - let workspace_id = operation.workspace_id.clone(); - let workdirs = self - .execute_store_stage(move |store| { - store.list_workdir_registry(&workspace_id, i64::MAX as usize) - }) - .await?; - for workdir in workdirs { - let removal_api = api.clone(); - let workdir_id = workdir.workdir_id.clone(); - let parent_id = operation_id.to_owned(); - match api - .host_executors - .host()? - .run(move || { - execute_workdir_removal_for_workspace_deletion( - &removal_api, - &workdir_id, - &parent_id, - ) - }) - .await - { + for workdir in self + .store + .list_workdir_registry(&operation.workspace_id, i64::MAX as usize)? + { + match execute_workdir_removal_for_workspace_deletion( + &api, + &workdir.workdir_id, + operation_id, + ) { Ok(child) => { - let id = operation_id.to_owned(); - let child_id = child.operation_id.clone(); child_operation_ids = self - .execute_store_stage(move |store| { - store.append_workspace_deletion_child_operation(&id, &child_id) - }) - .await? + .store + .append_workspace_deletion_child_operation( + operation_id, + &child.operation_id, + )? .child_operation_ids; if child.state != WorkdirRemovalOperationState::Completed || child.disposition != Some(WorkdirRemovalDisposition::Removed) @@ -2571,40 +2430,29 @@ impl WorkspaceServerApi { } if !blockers.is_empty() { - let id = operation_id.to_owned(); - return self - .execute_store_stage(move |store| { - store.update_workspace_deletion_operation( - &id, - WorkspaceDeletionState::Blocked, - &child_operation_ids, - &blockers, - None, - ) - }) - .await; + return self.store.update_workspace_deletion_operation( + operation_id, + WorkspaceDeletionState::Blocked, + &child_operation_ids, + &blockers, + None, + ); } api.purge_drive().await?; let feature_storage = api.feature_storage.clone(); - api.store - .db_executor() - .run(move || { - feature_storage.delete().map_err(|error| { - Error::Store(format!( - "failed to delete Workspace Feature storage: {error}" - )) - }) - }) - .await?; - let materials = self.signing_materials.clone(); - let id = operation_id.to_owned(); - let completed = self - .execute_store_stage(move |store| { - WorkspaceSigningIdentityService::new(store.clone(), materials) - .delete_material(&operation.workspace_id)?; - store.finalize_workspace_deletion(&id) - }) - .await?; + tokio::task::spawn_blocking(move || feature_storage.delete()) + .await + .map_err(|error| { + Error::Store(format!("Feature storage deletion task failed: {error}")) + })? + .map_err(|error| { + Error::Store(format!( + "failed to delete Workspace Feature storage: {error}" + )) + })?; + WorkspaceSigningIdentityService::new(self.store.clone(), self.signing_materials.clone()) + .delete_material(&operation.workspace_id)?; + let completed = self.store.finalize_workspace_deletion(operation_id)?; self.routers.lock().await.remove(&completed.workspace_id); if let Some(handle) = self .hook_handles @@ -2651,28 +2499,15 @@ impl WorkspaceServerApi { async fn shutdown_feature_storage(&self) -> Result<()> { let apis = self.apis.lock().await.values().cloned().collect::>(); for api in apis { - api.worker_projection.shutdown_async().await; - let storage = api.feature_storage.clone(); - api.store - .db_executor() - .run(move || { - storage.shutdown().map_err(|error| { - Error::Store(format!("failed to shut down Feature storage: {error}")) - }) - }) - .await?; + api.feature_storage.shutdown().map_err(|error| { + Error::Store(format!("failed to shut down Feature storage: {error}")) + })?; } Ok(()) } async fn preload(&self) -> Result<()> { - let store = self.store.clone(); - let workspaces = self - .store - .db_executor() - .run(move || store.list_workspaces()) - .await?; - for workspace in workspaces { + for workspace in self.store.list_workspaces()? { let _ = self .router_for_workspace(&workspace.workspace_id) .await? @@ -2742,7 +2577,7 @@ async fn resolve_server_actor( headers: &HeaderMap, ) -> std::result::Result, Error> { let cookie_name = auth_public_config(api.template.as_ref()).cookie_name; - resolve_request_actor(api.store.clone(), headers, &cookie_name).await + resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await } fn signed_request_target(uri: &Uri) -> &str { @@ -2781,11 +2616,6 @@ fn runtime_request_proof_rejection( workspace_id: &str, error: &crate::worker_source::RuntimeRequestProofError, ) -> Response { - let status = if error.is_availability_error() { - StatusCode::SERVICE_UNAVAILABLE - } else { - StatusCode::UNAUTHORIZED - }; // Never format an error or raw claims. IDs are fixed-length fingerprints; // verification labels describe proof-check progress, not authorization. let path = path.split('?').next().unwrap_or(path); @@ -2797,7 +2627,7 @@ fn runtime_request_proof_rejection( target: "yoi::auth", event = "runtime_request_proof_rejected", method, path = log_path, workspace_id, request_id, - status = status.as_u16(), + status = 401, reason = error.reason, proof_verification = error.verification, claimed_runtime_id_hash = error.runtime_id_hash.as_deref(), @@ -2812,12 +2642,8 @@ fn runtime_request_proof_rejection( ); repository_api_rejection( path, - status, - if error.is_availability_error() { - "Runtime authentication authority unavailable" - } else { - "invalid runtime request proof" - }, + StatusCode::UNAUTHORIZED, + "invalid runtime request proof", ) } @@ -2892,7 +2718,7 @@ async fn authorize_scoped_workspace_request( worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION }; let source = crate::worker_source::verify_runtime_request_source_proof_with_clock( - api.store.clone(), + api.store.as_ref(), api.template.as_ref(), &proof, workspace_id, @@ -3057,38 +2883,41 @@ async fn authorize_workspace_api_request( if !matches!( *request.method(), Method::GET | Method::HEAD | Method::OPTIONS - ) { - let workspace = match api.store.get_workspace(&workspace_id).await { - Ok(workspace) => workspace, - Err(error) => return server_error_response(error), - }; - if !workspace.is_some_and(|workspace| workspace.state == "active") { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); - } + ) && !api + .store + .get_workspace(&workspace_id) + .await + .ok() + .flatten() + .is_some_and(|workspace| workspace.state == "active") + { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); } return next.run(request).await; } let AuthConfig::Passkey { cookie_name, .. } = &api.config.auth; - let actor = - match crate::auth::resolve_request_actor(api.store.clone(), request.headers(), cookie_name) - .await - { - Ok(Some(actor)) => actor, - Ok(None) if public_server_api => return next.run(request).await, - Err(error) => return server_error_response(error), - Ok(None) => { - return repository_api_rejection( - &request_path, - StatusCode::UNAUTHORIZED, - "authentication required", - ); - } - }; + let actor = match crate::auth::resolve_request_actor( + api.store.as_ref(), + request.headers(), + cookie_name, + ) + .await + { + Ok(Some(actor)) => actor, + Ok(None) if public_server_api => return next.run(request).await, + Ok(None) | Err(_) => { + return repository_api_rejection( + &request_path, + StatusCode::UNAUTHORIZED, + "authentication required", + ); + } + }; let cookie_authenticated = matches!(actor.auth_method, ActorAuthMethod::BrowserSession); let mutating = !matches!( *request.method(), @@ -3111,18 +2940,20 @@ async fn authorize_workspace_api_request( } } request.extensions_mut().insert(actor); - if mutating { - let workspace = match api.store.get_workspace(&workspace_id).await { - Ok(workspace) => workspace, - Err(error) => return server_error_response(error), - }; - if !workspace.is_some_and(|workspace| workspace.state == "active") { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); - } + if mutating + && !api + .store + .get_workspace(&workspace_id) + .await + .ok() + .flatten() + .is_some_and(|workspace| workspace.state == "active") + { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); } next.run(request).await } @@ -3173,7 +3004,7 @@ fn workspace_request_is_callback(method: &Method, path: &str, workspace_id: &str async fn dispatch_workspace_request( State(api): State, - request: Request, + mut request: Request, ) -> Response { let path = request.uri().path().to_owned(); let workspace_id = scoped_workspace_id(&path); @@ -3199,23 +3030,6 @@ async fn dispatch_workspace_request( } else { None }; - if let Some(lease) = _request_lease { - crate::db_executor::with_operation_lifetime( - Arc::new(lease), - dispatch_admitted_workspace_request(api, request), - ) - .await - } else { - dispatch_admitted_workspace_request(api, request).await - } -} - -async fn dispatch_admitted_workspace_request( - api: WorkspaceServerApi, - mut request: Request, -) -> Response { - let path = request.uri().path().to_owned(); - let workspace_id = scoped_workspace_id(&path); if let Some(workspace_id) = workspace_id && (path.starts_with("/api/w/") || path.starts_with("/api/runtime/v1/workspaces/") @@ -3232,13 +3046,7 @@ async fn dispatch_admitted_workspace_request( Err(error) => return server_error_response(error), } } else { - let store = api.store.clone(); - let workspaces = match api - .store - .db_executor() - .run(move || store.list_workspaces()) - .await - { + let workspaces = match api.store.list_workspaces() { Ok(workspaces) => workspaces, Err(error) => return server_error_response(error), }; @@ -3514,19 +3322,6 @@ impl WorkspaceApi { } pub async fn new(config: ServerConfig, store: Arc) -> Result { - Self::new_with_host_executors(config, store, HostExecutors::shared()).await - } - - #[cfg(test)] - async fn new_for_test(config: ServerConfig, store: Arc) -> Result { - Self::new_with_host_executors(config, store, HostExecutors::default()).await - } - - async fn new_with_host_executors( - config: ServerConfig, - store: Arc, - host_executors: HostExecutors, - ) -> Result { let resource_broker = BackendResourceBroker::default(); if store.get_workspace(&config.workspace_id).await?.is_none() { return Err(crate::Error::Config(format!( @@ -3603,7 +3398,6 @@ impl WorkspaceApi { store, Arc::new(execution_backend), resource_broker, - host_executors, Some(worker_remove_dispatcher), ) .await?; @@ -3664,7 +3458,6 @@ impl WorkspaceApi { store, execution_backend, BackendResourceBroker::default(), - HostExecutors::default(), None, ) .await @@ -3675,7 +3468,6 @@ impl WorkspaceApi { store: Arc, execution_backend: Arc, resource_broker: BackendResourceBroker, - host_executors: HostExecutors, worker_remove_dispatcher: Option< Arc, >, @@ -3686,18 +3478,8 @@ impl WorkspaceApi { config.workspace_id ))); } - let repository_store = store.clone(); - let repository_config = config.clone(); - config.repositories = store - .db_executor() - .run(move || { - import_configured_repositories(repository_store.as_ref(), &repository_config)?; - load_configured_repositories_from_store( - repository_store.as_ref(), - &repository_config, - ) - }) - .await?; + import_configured_repositories(store.as_ref(), &config)?; + config.repositories = load_configured_repositories_from_store(store.as_ref(), &config)?; let embedded_runtime = EmbeddedWorkerRuntime::new_fs_store_with_execution_backend( config.workspace_id.clone(), config.embedded_runtime_store_root.clone(), @@ -3712,34 +3494,20 @@ impl WorkspaceApi { RuntimeSubscriptionBroker::new(config.workspace_id.clone()); runtime_subscription_broker .register_embedded_runtime(embedded_runtime_id, embedded_subscription_runtime); - let database_path = config.database_path.clone(); - let executor = store.db_executor(); - let config_executor = executor.clone(); - let (config_store, repository_secrets) = executor - .run(move || { - let config_store = Arc::new( - crate::SqliteWorkspaceStore::open(database_path.clone())? - .with_db_executor(config_executor), - ); - let repository_secrets = Arc::new(RepositorySecretService::open( - config_store.clone(), - &database_path, - )?); - Ok::<_, Error>((config_store, repository_secrets)) - }) - .await?; + let config_store = Arc::new(crate::SqliteWorkspaceStore::open( + config.database_path.clone(), + )?); + let repository_secrets = Arc::new(RepositorySecretService::open( + config_store.clone(), + &config.database_path, + )?); let signing_materials: Arc = Arc::new(FsWorkspaceSigningMaterialStore::new( workspace_signing_material_root(&config.database_path), )); let signing_identities = WorkspaceSigningIdentityService::new(store.clone(), signing_materials); - let identities = signing_identities.clone(); - let workspace_id = config.workspace_id.clone(); - store - .db_executor() - .run(move || identities.get_validated(&workspace_id)) - .await?; + signing_identities.get_validated(&config.workspace_id)?; let backend_url = config .backend_base_url .clone() @@ -3782,19 +3550,11 @@ impl WorkspaceApi { .with_provider(Arc::new(RuntimeConfigSchemaProvider)) .with_provider(Arc::new(RepositoryAccessConfigSchemaProvider)) .with_provider(Arc::new(skills::SkillConfigSchemaProvider)); - let materialization_store = config_store.clone(); - let workspace_id = config.workspace_id.clone(); - let schema = config_schema_registry.compose()?; - store - .db_executor() - .run(move || { - materialization_store.ensure_workspace_config_materialized_with_schema( - &workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), - schema, - ) - }) - .await?; + config_store.ensure_workspace_config_materialized_with_schema( + &config.workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + config_schema_registry.compose()?, + )?; let worker_projection = crate::worker_projection::WorkerProjectionService::new( config.workspace_id.clone(), store.clone(), @@ -3805,28 +3565,18 @@ impl WorkspaceApi { .await .map_err(|error| Error::InvalidInput(error.to_string()))?; } - let database_path = config.database_path.clone(); - let workspace_id = config.workspace_id.clone(); - let (feature_storage, subjektiv_registration, authority) = store - .db_executor() - .run(move || { - let feature_storage = crate::FeatureStorage::for_server_database(&database_path) - .and_then(|storage| storage.workspace(&workspace_id)) - .map_err(|error| { - Error::Store(format!("failed to initialize Feature storage: {error}")) - })?; - let registration = crate::subjektiv::SubjektivStore::register(&feature_storage) - .map_err(|error| { - Error::Store(format!("failed to register subjektiv storage: {error}")) - })?; - let authority = SqliteWorkspaceAuthority::new(database_path, workspace_id)?; - Ok::<_, Error>((feature_storage, registration, authority)) - }) - .await?; + let feature_storage = crate::FeatureStorage::for_server_database(&config.database_path) + .and_then(|storage| storage.workspace(&config.workspace_id)) + .map_err(|error| { + Error::Store(format!("failed to initialize Feature storage: {error}")) + })?; + let subjektiv_registration = crate::subjektiv::SubjektivStore::register(&feature_storage) + .map_err(|error| { + Error::Store(format!("failed to register subjektiv storage: {error}")) + })?; let drive_host = Arc::new(crate::workspace_drive_host::WorkspaceDriveHost::new(&feature_storage).await?); let mut api = Self { - host_executors, drive_host, config_store, repository_secrets, @@ -3836,7 +3586,10 @@ impl WorkspaceApi { crate::prompt_settings::WorkspacePromptProjectionCache::default(), feature_storage, subjektiv_registration, - authority, + authority: SqliteWorkspaceAuthority::new( + config.database_path.clone(), + config.workspace_id.clone(), + )?, _worker_projection_shutdown: Arc::new(WorkerProjectionShutdownGuard { service: Arc::downgrade(&worker_projection), }), @@ -3886,88 +3639,62 @@ impl WorkspaceApi { )) .map_err(|message| Error::Config(message.to_string()))?; } - let recovery_api = api.clone(); - let expired_external_workdirs = api - .store - .db_executor() - .run(move || { - recovery_api - .store - .reconcile_external_workdir_grants_after_restart( - &recovery_api.config.workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - }) - .await?; + let expired_external_workdirs = api.store.reconcile_external_workdir_grants_after_restart( + &api.config.workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; for workdir_id in expired_external_workdirs { cleanup_external_workdir_attachments(&api, &workdir_id) .await .map_err(|error| error.error)?; } - let grant_api = api.clone(); - let grants = api + for grant in api .store - .db_executor() - .run(move || { - grant_api - .store - .list_external_workdir_grants(&grant_api.config.workspace_id) - }) - .await?; - for grant in grants + .list_external_workdir_grants(&api.config.workspace_id)? .into_iter() .filter(|grant| !matches!(grant.status.as_str(), "revoked" | "expired")) { schedule_external_workdir_expiry(&api, &grant)?; } - let workdir_recovery_api = api.clone(); - api.host_executors - .host()? - .run(move || recover_workdir_removals(&workdir_recovery_api)) - .await?; + recover_workdir_removals(&api)?; recover_runtime_removals(&api).await; - api.host_executors - .host()? - .run(move || { - let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let deliveries = api - .store - .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; - for delivery in deliveries { - let Some(job) = api - .store - .get_backend_job(&api.config.workspace_id, &delivery.job_id)? - else { - continue; - }; - let Some(attempt) = api.store.get_backend_job_attempt( - &api.config.workspace_id, - &delivery.job_id, - &delivery.attempt_id, - )? - else { - continue; - }; - if job.state == BackendJobState::Completed - && attempt.state == BackendJobAttemptState::Completed - { - api.deliver_backend_job_result( - &BackendJobResultAcceptance { - job, - attempt, - replayed: true, - }, - &delivery.target, - ); - } - } - api.recover_backend_jobs()?; - if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { - dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); - } - Ok(api) - }) - .await + let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let deliveries = api + .store + .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; + for delivery in deliveries { + let Some(job) = api + .store + .get_backend_job(&api.config.workspace_id, &delivery.job_id)? + else { + continue; + }; + let Some(attempt) = api.store.get_backend_job_attempt( + &api.config.workspace_id, + &delivery.job_id, + &delivery.attempt_id, + )? + else { + continue; + }; + if job.state == BackendJobState::Completed + && attempt.state == BackendJobAttemptState::Completed + { + api.deliver_backend_job_result( + &BackendJobResultAcceptance { + job, + attempt, + replayed: true, + }, + &delivery.target, + ); + } + } + api.recover_backend_jobs()?; + if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + Ok(api) } pub fn workspace_id(&self) -> &str { @@ -4360,44 +4087,77 @@ impl WorkspaceApi { { return false; } - let Ok(runtime) = tokio::runtime::Handle::try_current() else { - tracing::warn!(job_id = %attempt.job_id, "Backend Job cleanup remains durable without a runtime"); + let Ok(handle) = tokio::runtime::Handle::try_current() else { + tracing::warn!( + job_id = %attempt.job_id, + attempt_id = %attempt.attempt_id, + "Backend Job Worker cleanup remains durable because no Tokio runtime is available" + ); return false; }; + let (attempt, claimed) = match self.store.claim_backend_job_worker_cleanup( + &self.config.workspace_id, + &attempt.job_id, + &attempt.attempt_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) { + Ok(claim) => claim, + Err(error) => { + tracing::warn!( + %error, + job_id = %attempt.job_id, + attempt_id = %attempt.attempt_id, + "failed to claim durable Backend Job Worker cleanup" + ); + return false; + } + }; + if !claimed { + return false; + } + let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed + || attempt.failure_category.as_deref() == Some("invalid_result"); + if waits_for_run_teardown { + let api = self.clone(); + let cleanup_handle = handle.clone(); + handle.spawn(async move { + if let Err(error) = api.wait_for_backend_job_worker_run_teardown(&attempt).await { + tracing::warn!( + %error, + job_id = %attempt.job_id, + attempt_id = %attempt.attempt_id, + "Backend Job Worker Run teardown fence failed" + ); + return; + } + api.spawn_claimed_backend_job_worker_cleanup(&cleanup_handle, attempt); + }); + } else { + self.spawn_claimed_backend_job_worker_cleanup(&handle, attempt); + } + true + } + + fn spawn_claimed_backend_job_worker_cleanup( + &self, + handle: &tokio::runtime::Handle, + attempt: BackendJobAttemptRecord, + ) { let api = self.clone(); let job_id = attempt.job_id.clone(); let attempt_id = attempt.attempt_id.clone(); - // Admit before claiming: overload must leave durable Pending/Failed - // ownership recoverable, never strand a Running claim without a job. - let admission = self.host_executors.cleanup().and_then(|executor| { - executor.dispatch(move || { + handle.spawn_blocking(move || { let cleanup = || -> Result<()> { - let (attempt, claimed) = api.store.claim_backend_job_worker_cleanup( - &api.config.workspace_id, &job_id, &attempt_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; - if !claimed { return Ok(()); } - let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed - || attempt.failure_category.as_deref() == Some("invalid_result"); - runtime.block_on(async { - if waits_for_run_teardown { - api.wait_for_backend_job_worker_run_teardown(&attempt).await?; - } - // Already on the Host owner; removal must not submit to and - // wait on its own lane. All transaction helpers return before I/O. - api.cleanup_backend_job_attempt_worker(attempt).await - }) + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| Error::Store(error.to_string()))? + .block_on(api.cleanup_backend_job_attempt_worker(attempt)) }; if let Err(error) = cleanup() { tracing::warn!(%error, %job_id, %attempt_id, "Backend Job Worker cleanup failed"); } - }) }); - if let Err(error) = admission { - tracing::warn!(%error, "Backend Job cleanup was not admitted; its durable ownership is unchanged"); - return false; - } - true } async fn wait_for_backend_job_worker_run_teardown( @@ -4532,7 +4292,7 @@ impl WorkspaceApi { // recover that midpoint; without such an operation this remains // a diagnosable retryable failure rather than a direct DB delete. let response = WorkerRemovalService::new(self) - .execute_target_removal_on_host( + .execute_target_removal( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4598,7 +4358,7 @@ impl WorkspaceApi { } let response = WorkerRemovalService::new(self) - .execute_target_removal_on_host( + .execute_target_removal( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4725,7 +4485,6 @@ impl WorkspaceApi { let attempt_id = attempt_id.to_string(); handle.spawn(async move { tokio::time::sleep(delay).await; - let admission = api.host_executors.host().and_then(|executor| executor.dispatch(move || { let Ok(Some(attempt)) = api.store .get_backend_job_attempt(&api.config.workspace_id, &job_id, &attempt_id) @@ -4749,10 +4508,6 @@ impl WorkspaceApi { { api.drain_reserved_backend_jobs(); } - })); - if let Err(error) = admission { - tracing::warn!(%error, "Backend Job timeout was not admitted; its deadline remains durable"); - } }); } @@ -6470,7 +6225,7 @@ async fn attach_server_request_context( } else { let auth = service.auth_api(); match resolve_request_actor( - auth.store.clone(), + auth.store.as_ref(), request.headers(), &auth_public_config(&auth.config).cookie_name, ) @@ -7436,12 +7191,7 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyRegistrationComplete, server_api::RepositoryApiError, > { - let api = self.auth_api(); - let (body, header_set_cookie) = api - .store - .db_executor() - .run(move || complete_passkey_registration(&api, request)) - .await + let (body, header_set_cookie) = complete_passkey_registration(&self.auth_api(), request) .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyRegistrationComplete::Status200 { @@ -7472,12 +7222,7 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyLoginComplete, server_api::RepositoryApiError, > { - let api = self.auth_api(); - let (body, header_set_cookie) = api - .store - .db_executor() - .run(move || complete_passkey_login(&api, request)) - .await + let (body, header_set_cookie) = complete_passkey_login(&self.auth_api(), request) .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyLoginComplete::Status200 { @@ -7510,11 +7255,7 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - let api = self.auth_api(); - api.store - .db_executor() - .run(move || approve_device_login(&api, actor, request)) - .await + approve_device_login(&self.auth_api(), actor, request) .map_err(ApiError::into_repository_api_error) } @@ -7544,12 +7285,7 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthLogout, server_api::RepositoryApiError, > { - let api = self.auth_api(); - let (body, header_set_cookie) = api - .store - .db_executor() - .run(move || logout_browser_session(&api, cookie)) - .await + let (body, header_set_cookie) = logout_browser_session(&self.auth_api(), cookie) .map_err(ApiError::into_repository_api_error)?; Ok(server_api::server_api_responses::AuthLogout::Status200 { body, @@ -7564,12 +7300,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.server_api()?; let Some(actor) = context.actor else { - let catalog = api.catalog.clone(); return if api - .store - .db_executor() - .run(move || catalog.is_empty()) - .await + .catalog + .is_empty() .map_err(|error| ApiError::from(error).into_repository_api_error())? { Ok(WorkspaceCatalogListResponse(Vec::new())) @@ -7582,17 +7315,12 @@ impl server_api::ServerApi for ServerApiContractService { )) }; }; - let catalog = api.catalog.clone(); let workspaces = api - .store - .db_executor() - .run(move || { - catalog.list( - &actor.account_id, - query.limit.unwrap_or(100).try_into().unwrap_or(200), - ) - }) - .await + .catalog + .list( + &actor.account_id, + query.limit.unwrap_or(100).try_into().unwrap_or(200), + ) .map_err(|error| ApiError::from(error).into_repository_api_error())?; Ok(WorkspaceCatalogListResponse( workspaces.into_iter().map(workspace_summary).collect(), @@ -7613,12 +7341,9 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - let catalog = api.catalog.clone(); let created = api - .store - .db_executor() - .run(move || catalog.create(request, actor.account_id)) - .await + .catalog + .create(request, actor.account_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; api.router_for_workspace(&created.workspace.workspace_id) .await @@ -7663,14 +7388,10 @@ impl server_api::ServerApi for ServerApiContractService { ) })?; let admission_for_workspace = api.admission_for_workspace(&workspace_id).await; - let mutation_guard = Arc::new(admission_for_workspace.lock_deletion().await); - let account_id = actor.account_id.clone(); - let operation_id = request.operation_id.clone(); + let _mutation_guard = admission_for_workspace.lock_deletion().await; let existing = api - .execute_store_stage(move |store| { - store.workspace_deletion_operation(&account_id, &operation_id) - }) - .await + .store + .workspace_deletion_operation(&actor.account_id, &request.operation_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; if existing.is_none() { let preflight = api @@ -7686,18 +7407,9 @@ impl server_api::ServerApi for ServerApiContractService { )); } } - let account_id = actor.account_id.clone(); - let id = workspace_id.clone(); - let reserve_request = request.clone(); - let reserve_guard = mutation_guard.clone(); let reservation = api - .execute_store_stage(move |store| { - // Keep deletion closed until the accepted reservation really finishes, - // even when the HTTP task drops its original guard. - let _guard = reserve_guard; - store.reserve_workspace_deletion(&account_id, &id, &reserve_request) - }) - .await + .store + .reserve_workspace_deletion(&actor.account_id, &workspace_id, &request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; if let Some(handle) = api.hook_handles.lock().await.remove(&workspace_id) { handle.abort(); @@ -7737,19 +7449,17 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - api.execute_store_stage(move |store| { - store.workspace_deletion_operation(&actor.account_id, &operation_id) - }) - .await - .map_err(|error| ApiError::from(error).into_repository_api_error())? - .ok_or_else(|| { - server_api::RepositoryApiError::new( - StatusCode::NOT_FOUND.as_u16(), - "Not Found", - "Workspace deletion operation not found", - Vec::new(), - ) - }) + api.store + .workspace_deletion_operation(&actor.account_id, &operation_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())? + .ok_or_else(|| { + server_api::RepositoryApiError::new( + StatusCode::NOT_FOUND.as_u16(), + "Not Found", + "Workspace deletion operation not found", + Vec::new(), + ) + }) } async fn workspace_current( @@ -7784,20 +7494,13 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::WorkspaceRuntimeConfig, server_api::RepositoryApiError, > { - let api = self.workspace_api()?.clone(); - api.store - .db_executor() - .run(move || { - Ok::<_, Error>(workspace_runtime_config_result( - &api, - context, - &workspace_id, - query, - if_none_match, - )) - }) - .await - .map_err(|error| ApiError::from(error).into_repository_api_error())? + workspace_runtime_config_result( + self.workspace_api()?, + context, + &workspace_id, + query, + if_none_match, + ) } async fn workspace_metadata_settings( @@ -9690,7 +9393,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; restore_runtime_worker_with_context(api, runtime_id, worker_id, operation_context, request) .await .map(|Json(response)| response) @@ -9708,7 +9411,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; scoped_restore_runtime_worker_with_context( @@ -9735,11 +9438,10 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, true) @@ -9756,11 +9458,10 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, false) @@ -9797,7 +9498,7 @@ impl server_api::ServerApi for ServerApiContractService { server_api::RepositoryApiError, > { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; execute_runtime_cleanup_with_context(&api, &runtime_id, request, &operation_context) @@ -9814,9 +9515,8 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -9837,11 +9537,10 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -10001,9 +9700,8 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -10025,11 +9723,10 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -10050,9 +9747,8 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -10074,11 +9770,10 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let operation_context = WorkerOperationContext::from_request(&api, &context)?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) - .await + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -10893,28 +10588,23 @@ impl server_api::ServerApi for ServerApiContractService { query: server_api::TicketAssignmentsQuery, ) -> std::result::Result { - let api = self.workspace_api()?.clone(); + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - api.store - .page_ticket_assignments( - &workspace_id, - &ticket_id, - query.after_assignment_id.as_deref(), - query.limit.unwrap_or(30), - ) - .map(server_api::TicketAssignmentsPage) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error) - }) - .await + .page_ticket_assignments( + &workspace_id, + &ticket_id, + query.after_assignment_id.as_deref(), + query.limit.unwrap_or(30), + ) + .map(server_api::TicketAssignmentsPage) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error) } async fn ticket_assignments_page_browser( &self, @@ -10937,29 +10627,24 @@ impl server_api::ServerApi for ServerApiContractService { server_api::TicketAssignmentDiagnosticsPage, server_api::RepositoryApiError, > { - let api = self.workspace_api()?.clone(); + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - api.store - .page_ticket_assignment_diagnostics( - &workspace_id, - &ticket_id, - &query.assignment_id, - query.after_input_event_id.as_deref(), - query.after_start_operation_id.as_deref(), - query.limit.unwrap_or(30), - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error) - }) - .await + .page_ticket_assignment_diagnostics( + &workspace_id, + &ticket_id, + &query.assignment_id, + query.after_input_event_id.as_deref(), + query.after_start_operation_id.as_deref(), + query.limit.unwrap_or(30), + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error) } async fn ticket_assignment_diagnostics_browser( &self, @@ -10984,74 +10669,26 @@ impl server_api::ServerApi for ServerApiContractService { server_api::TicketAssignmentAnswerResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?.clone(); - // Runtime discovery may perform remote I/O. Preserve the live singleton - // check on the finite Host lane; the answer's source/binding authority - // is checked again under its IMMEDIATE transaction on the DB lane. - if context.runtime_source.is_some() { - let source_api = api.clone(); - let source_context = context.clone(); - let source_workspace_id = workspace_id.clone(); - api.host_executors - .host() - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)? - .run(move || { - validate_workspace_scope(&source_api, &source_workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - authenticated_assignment_source( - &source_api, - &contract_request_headers(&source_context)?, - &source_context, - ) - .map(|_| ()) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error) - }) - .await?; - } - let (receipt, notification) = api - .store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let (actor, source) = assignment_source_from_authenticated_context( - &contract_request_headers(&context)?, - &context, - ) + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let (actor, source) = + authenticated_assignment_source(api, &contract_request_headers(&context)?, &context) .map_err(ApiError::from) .map_err(ApiError::into_repository_api_error)?; - let receipt = api - .store - .answer_ticket_assignment( - &workspace_id, - &ticket_id, - &request.0, - &source, - &actor, - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - // This admitted SQL job owns source/fence/CAS/event/audit/receipt - // and commit. Admit its post-commit Host continuation here, not in - // the cancellable waiter. It inherits the request lifecycle lease; - // Runtime/network dispatch never occupies the DB worker. - Ok::<_, server_api::RepositoryApiError>(( - receipt, - enqueue_orchestrator_assignment_attention(&api), - )) - }) - .await?; - if let Some(notification) = notification { - // Preserve normal best-effort notification completion for a live - // response. Losing this receiver cannot cancel the accepted job. - let _ = notification.await; + let receipt = api + .store + .answer_ticket_assignment(&workspace_id, &ticket_id, &request.0, &source, &actor) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + // The entire event/audit/receipt transaction has committed. Response loss + // retries the saved receipt; delivery scan deduplicates accepted inputs. + if let Some(orchestrator) = find_workspace_orchestrator(api) { + dispatch_orchestrator_assignment_attention(api, &orchestrator.worker); } Ok(server_api::TicketAssignmentAnswerResponse(receipt)) } @@ -11077,28 +10714,23 @@ impl server_api::ServerApi for ServerApiContractService { query: server_api::TicketAssignmentTrackingQuery, ) -> std::result::Result { - let api = self.workspace_api()?.clone(); - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let values = if let Some(assignment_id) = query.assignment_id { - api.store - .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) - .map(|value| value.into_iter().collect::>()) - } else { - api.authority.ticket_assignment_tracking(&ticket_id) - } - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - Ok(server_api::TicketAssignmentTrackingList(values)) - }) - .await + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let values = if let Some(assignment_id) = query.assignment_id { + api.store + .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) + .map(|value| value.into_iter().collect::>()) + } else { + api.authority.ticket_assignment_tracking(&ticket_id) + } + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + Ok(server_api::TicketAssignmentTrackingList(values)) } async fn ticket_assignment_tracking_update( @@ -11925,30 +11557,21 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirCatalogResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?.clone(); - let headers = current_worker_contract_headers(&context)?; - let stage_api = api.clone(); - let worker = api - .host_executors - .host() - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)? - .run(move || -> ApiResult<_> { - validate_workspace_scope(&stage_api, &workspace_id)?; - Ok(current_worker_identity( - &stage_api, - &workspace_id, - &headers, - )?) - }) - .await + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let session_lock = current_worker_session_lock(&api, &worker); - let guard = Arc::new(session_lock.lock_owned().await); - run_worker_workdir_db_stage_locked(&api, &guard, move |api| { - list_current_worker_workdir_catalog_for_worker(&api, &worker, query) - }) - .await + let worker = current_worker_identity( + api, + &workspace_id, + ¤t_worker_contract_headers(&context)?, + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + { + let session_lock = current_worker_session_lock(api, &worker); + let _guard = session_lock.lock().await; + list_current_worker_workdir_catalog_for_worker(api, &worker, query) + } .map_err(ApiError::into_repository_api_error) } @@ -11961,31 +11584,20 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirAttachmentListResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?.clone(); - let headers = current_worker_contract_headers(&context)?; - let stage_api = api.clone(); - let worker = api - .host_executors - .host() - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)? - .run(move || -> ApiResult<_> { - validate_workspace_scope(&stage_api, &workspace_id)?; - Ok(current_worker_identity( - &stage_api, - &workspace_id, - &headers, - )?) - }) - .await + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let session_lock = current_worker_session_lock(&api, &worker); - let guard = Arc::new(session_lock.lock_owned().await); - run_worker_workdir_db_stage_locked(&api, &guard, move |api| { - list_current_worker_workdir_attachments(&api, &worker, query) - }) - .await - .map_err(ApiError::into_repository_api_error) + let worker = current_worker_identity( + api, + &workspace_id, + ¤t_worker_contract_headers(&context)?, + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let session_lock = current_worker_session_lock(api, &worker); + let _session_guard = session_lock.lock().await; + list_current_worker_workdir_attachments(api, &worker, query) + .map_err(ApiError::into_repository_api_error) } async fn current_worker_workdir_attach( @@ -12687,13 +12299,8 @@ async fn scoped_list_flows( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult>> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) } async fn scoped_put_flow( @@ -12701,34 +12308,28 @@ async fn scoped_put_flow( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let definition = flow::compile_flow_source(&request.content).map_err(|error| { - Error::InvalidInput(format!( - "invalid Flow source: {}", - serde_json::to_string(&error.diagnostics) - .unwrap_or_else(|_| "diagnostics unavailable".to_string()) - )) - })?; - let expected_path = format!("flows/{}.dcdl", definition.name); - if request.path != expected_path { - return Err(Error::InvalidInput(format!( - "Flow source path must be `{expected_path}`" - )) - .into()); - } - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); - Ok(Json(api.store.put_flow_source_for_kind( - &path.workspace_id, - FlowSourceKind::Workspace, - &request.path, - &request.content, - &now, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let definition = flow::compile_flow_source(&request.content).map_err(|error| { + Error::InvalidInput(format!( + "invalid Flow source: {}", + serde_json::to_string(&error.diagnostics) + .unwrap_or_else(|_| "diagnostics unavailable".to_string()) + )) + })?; + let expected_path = format!("flows/{}.dcdl", definition.name); + if request.path != expected_path { + return Err( + Error::InvalidInput(format!("Flow source path must be `{expected_path}`")).into(), + ); + } + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + Ok(Json(api.store.put_flow_source_for_kind( + &path.workspace_id, + FlowSourceKind::Workspace, + &request.path, + &request.content, + &now, + )?)) } async fn scoped_resolve_flow_source( @@ -12736,79 +12337,65 @@ async fn scoped_resolve_flow_source( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = match &request.selector { - flow::FlowSelector::Builtin { slug } => { - let builtin = flow::builtin_flow_source(slug) - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let definition = builtin.compile().map_err(|error| { - Error::Store(format!( - "compile built-in Flow {slug:?}: {:?}", - error.diagnostics - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: format!("builtin:{slug}"), - content_digest: definition.content_digest.clone(), - definition, - } - } - flow::FlowSelector::Workspace { slug } => { - let source = api - .store - .get_flow_source_by_name( - &path.workspace_id, - FlowSourceKind::Workspace, - slug, - )? - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let content = api - .store - .get_flow_source_content( - &path.workspace_id, - &source.flow_id, - &source.content_digest, - )? - .ok_or_else(|| { - Error::Store(format!( - "resolved Flow content {}@{} is missing", - source.flow_id, source.content_digest - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: source.flow_id, - content_digest: content.content_digest, - definition: content.definition, - } - } - }; - Ok(Json(resolved)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let resolved = match &request.selector { + flow::FlowSelector::Builtin { slug } => { + let builtin = flow::builtin_flow_source(slug) + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let definition = builtin.compile().map_err(|error| { + Error::Store(format!( + "compile built-in Flow {slug:?}: {:?}", + error.diagnostics + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: format!("builtin:{slug}"), + content_digest: definition.content_digest.clone(), + definition, + } + } + flow::FlowSelector::Workspace { slug } => { + let source = api + .store + .get_flow_source_by_name(&path.workspace_id, FlowSourceKind::Workspace, slug)? + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let content = api + .store + .get_flow_source_content( + &path.workspace_id, + &source.flow_id, + &source.content_digest, + )? + .ok_or_else(|| { + Error::Store(format!( + "resolved Flow content {}@{} is missing", + source.flow_id, source.content_digest + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: source.flow_id, + content_digest: content.content_digest, + definition: content.definition, + } + } + }; + Ok(Json(resolved)) } async fn scoped_get_flow( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = api - .store - .get_flow_source(&path.workspace_id, &path.flow_id)? - .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; - Ok(Json(source)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let source = api + .store + .get_flow_source(&path.workspace_id, &path.flow_id)? + .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; + Ok(Json(source)) } async fn scoped_get_workspace_settings( @@ -12879,13 +12466,8 @@ async fn scoped_get_workspace_signing_identity( "Workspace public identity access", ) .await?; - api.store - .db_executor() - .run(move || { - let identity = api.signing_identities.get_validated(&path.workspace_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) - }) - .await + let identity = api.signing_identities.get_validated(&path.workspace_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) } async fn scoped_provision_workspace_signing_identity( @@ -12900,15 +12482,10 @@ async fn scoped_provision_workspace_signing_identity( "Workspace signing identity provisioning", ) .await?; - api.store - .db_executor() - .run(move || { - let identity = api - .signing_identities - .provision_existing(&path.workspace_id, &actor.account_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) - }) - .await + let identity = api + .signing_identities + .provision_existing(&path.workspace_id, &actor.account_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) } fn project_workspace_signing_identity( @@ -12986,19 +12563,12 @@ async fn scoped_get_workspace_config_history( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let snapshot = api - .config_store - .load_workspace_config_history(&path.workspace_id, &path.content_digest)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId(path.content_digest.clone())) - })?; - Ok(Json(config_tree_snapshot_to_api(snapshot))) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let snapshot = api + .config_store + .load_workspace_config_history(&path.workspace_id, &path.content_digest)? + .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.content_digest.clone())))?; + Ok(Json(config_tree_snapshot_to_api(snapshot))) } #[derive(Debug, Deserialize)] @@ -13011,44 +12581,34 @@ async fn scoped_get_prompt_projection( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidInput( - "Workspace config is not initialized".to_string(), - )) - })?; - let projection = api - .prompt_projection_cache - .resolve(&path.workspace_id, &state)?; - Ok(Json(projection.as_ref().clone())) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidInput( + "Workspace config is not initialized".to_string(), + )) + })?; + let projection = api + .prompt_projection_cache + .resolve(&path.workspace_id, &state)?; + Ok(Json(projection.as_ref().clone())) } async fn scoped_get_workspace_memory_settings( State(api): State, AxumPath(workspace_id): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .get_workspace_memory_settings(&workspace_id) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) - }) - .await + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .get_workspace_memory_settings(&workspace_id) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) } async fn scoped_update_workspace_memory_settings( @@ -13056,24 +12616,19 @@ async fn scoped_update_workspace_memory_settings( AxumPath(workspace_id): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .update_workspace_memory_settings( - &workspace_id, - &request.expected_language, - &request.language, - ) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) - }) - .await + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .update_workspace_memory_settings( + &workspace_id, + &request.expected_language, + &request.language, + ) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) } async fn require_workspace_owner( @@ -13584,51 +13139,41 @@ async fn scoped_get_workspace_config_tree( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { - snapshot: ConfigTreeSnapshot::empty(), - contract: config_source::ToolchainContract::new( - config_source::DEFAULT_SCHEMA_VERSION, - Vec::new(), - config_source::DEFAULT_IMPORT_POLICY_VERSION, - ), - projection_digest: config_source::digest_bytes(b"[]"), - }); - Ok(Json(workspace_config_state_to_api(state))) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { + snapshot: ConfigTreeSnapshot::empty(), + contract: config_source::ToolchainContract::new( + config_source::DEFAULT_SCHEMA_VERSION, + Vec::new(), + config_source::DEFAULT_IMPORT_POLICY_VERSION, + ), + projection_digest: config_source::digest_bytes(b"[]"), + }); + Ok(Json(workspace_config_state_to_api(state))) } async fn scoped_get_workspace_config_entry( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let virtual_path = config_source::VirtualPath::parse(&path.path) - .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - let entry = state - .snapshot - .get(&virtual_path) - .cloned() - .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; - Ok(Json(config_entry_to_api(entry))) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let virtual_path = config_source::VirtualPath::parse(&path.path) + .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + let entry = state + .snapshot + .get(&virtual_path) + .cloned() + .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; + Ok(Json(config_entry_to_api(entry))) } async fn scoped_commit_workspace_config_tree( @@ -13636,18 +13181,13 @@ async fn scoped_commit_workspace_config_tree( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - api.store - .db_executor() - .run(move || { - let request = config_commit_request_from_api(request)?; - validate_workspace_scope(&api, &path.workspace_id)?; - let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; - Ok(( - StatusCode::CREATED, - Json(workspace_config_state_to_api(state)), - )) - }) - .await + let request = config_commit_request_from_api(request)?; + validate_workspace_scope(&api, &path.workspace_id)?; + let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; + Ok(( + StatusCode::CREATED, + Json(workspace_config_state_to_api(state)), + )) } /// Shared canonical commit and activation boundary for UI and logical config operations. @@ -13711,25 +13251,20 @@ async fn scoped_get_profile_settings( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - Ok(Json( - crate::profile_settings::project_profiles_from_workspace_config( - &path.workspace_id, - &state, - )? - .settings, - )) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + Ok(Json( + crate::profile_settings::project_profiles_from_workspace_config( + &path.workspace_id, + &state, + )? + .settings, + )) } async fn scoped_list_tickets( @@ -13745,13 +13280,8 @@ async fn scoped_get_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - browser_ticket_detail(&api, &path.id) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + browser_ticket_detail(&api, &path.id) } async fn scoped_query_tickets( @@ -13759,13 +13289,8 @@ async fn scoped_query_tickets( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_tickets(query)?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_tickets(query)?)) } async fn scoped_show_ticket( @@ -13773,16 +13298,11 @@ async fn scoped_show_ticket( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let backend = browser_ticket_backend(&api)?; - Ok(Json(api.authority.read_ticket_detail_with_backend( - &path.id, query, &backend, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let backend = browser_ticket_backend(&api)?; + Ok(Json(api.authority.read_ticket_detail_with_backend( + &path.id, query, &backend, + )?)) } fn parse_ticket_assignment_role(role: &str) -> ApiResult { @@ -13858,24 +13378,19 @@ async fn scoped_list_ticket_assignments( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let ticket = api.authority.ticket(&path.id)?; - let assignments = api - .store - .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; - Ok(Json(server_api::TicketRoleAssignmentsResponse { - workspace_id: path.workspace_id, - ticket_id: ticket.id, - assignments: assignments - .into_iter() - .map(project_ticket_role_assignment_record) - .collect(), - })) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let ticket = api.authority.ticket(&path.id)?; + let assignments = api + .store + .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; + Ok(Json(server_api::TicketRoleAssignmentsResponse { + workspace_id: path.workspace_id, + ticket_id: ticket.id, + assignments: assignments + .into_iter() + .map(project_ticket_role_assignment_record) + .collect(), + })) } async fn compensate_manual_ticket_worker_binding( @@ -14014,31 +13529,6 @@ async fn start_manual_ticket_worker_assignment( operation_id: &str, selected: &[server_api::TicketWorkerAttachmentBinding], delegation: Option<&TicketStartDelegation>, -) -> ApiResult { - let api = api.clone(); - let record = record.clone(); - let operation_id = operation_id.to_owned(); - let selected = selected.to_vec(); - let delegation = delegation.cloned(); - run_host_workflow(api.host_executors.host()?, async move { - start_manual_ticket_worker_assignment_on_host( - &api, - &record, - &operation_id, - &selected, - delegation.as_ref(), - ) - .await - }) - .await -} - -async fn start_manual_ticket_worker_assignment_on_host( - api: &WorkspaceApi, - record: &TicketRoleAssignmentRecord, - operation_id: &str, - selected: &[server_api::TicketWorkerAttachmentBinding], - delegation: Option<&TicketStartDelegation>, ) -> ApiResult { let TicketAssignmentPrincipal::Worker { runtime_id, @@ -14455,35 +13945,13 @@ fn authenticated_assignment_source( String, crate::store::TicketOrchestratorAssignmentSourceContext, )> { - let result = assignment_source_from_authenticated_context(headers, context)?; - if let crate::store::TicketOrchestratorAssignmentSourceContext::Runtime { - runtime_id, - worker_id, - .. - } = &result.1 - { - let source = RuntimeWorkerRef::new(runtime_id, worker_id); + if let Some(source) = authenticated_ticket_worker_source(headers, context)? { if !find_workspace_orchestrator(api).is_some_and(|worker| worker.worker == source) { return Err(Error::WorkspacePermissionDenied( "Ticket assignment mutations require the registered current Workspace Orchestrator" .into(), )); } - } - Ok(result) -} - -// Attribution only: Backend authenticated this context, never body input. This -// performs no Runtime lookup. Store mutation helpers fence the current singleton -// and live authentication binding in the same transaction as CAS/audit/receipt. -fn assignment_source_from_authenticated_context( - headers: &HeaderMap, - context: &server_api::ServerRequestContext, -) -> Result<( - String, - crate::store::TicketOrchestratorAssignmentSourceContext, -)> { - if let Some(source) = authenticated_ticket_worker_source(headers, context)? { let authentication_binding_id = context .runtime_source .as_ref() @@ -14529,13 +13997,7 @@ async fn scoped_set_ticket_assignment_with_context( request: Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let source_api = api.clone(); - let context = context.clone(); - let (author, source) = api - .host_executors - .host()? - .run(move || authenticated_assignment_source(&source_api, &headers, &context)) - .await?; + let (author, source) = authenticated_assignment_source(&api, &headers, context)?; apply_set_ticket_assignment(State(api), path, request, author, Some(source)).await } @@ -14546,83 +14008,46 @@ async fn apply_set_ticket_assignment( assigned_by: String, source: Option, ) -> ApiResult> { - let prep_api = api.clone(); - let prep_workspace = workspace_id.clone(); - let (ticket, role, operation_id, expected_assignment_id, record, workdir_bindings) = api - .store - .db_executor() - .run(move || { - let api = &prep_api; - let workspace_id = prep_workspace; - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = - require_ticket_assignment_value("operation_id", request.operation_id)?; - let expected_assignment_id = request - .expected_assignment_id - .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) - .transpose()?; - let principal = internal_ticket_assignment_principal(request.principal)?; - let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let record = TicketRoleAssignmentRecord { - workspace_id: workspace_id.clone(), - ticket_id: ticket.id.clone(), - assignment_id: new_id("tasg"), - role, - principal, - assigned_by, - assigned_at, - }; - Ok::<_, ApiError>(( - ticket, - role, - operation_id, - expected_assignment_id, - record, - request.workdir_bindings, - )) - }) - .await?; - let (assignment, notification) = match role { + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = require_ticket_assignment_value("operation_id", request.operation_id)?; + let expected_assignment_id = request + .expected_assignment_id + .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) + .transpose()?; + let principal = internal_ticket_assignment_principal(request.principal)?; + let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let record = TicketRoleAssignmentRecord { + workspace_id: workspace_id.clone(), + ticket_id: ticket.id.clone(), + assignment_id: new_id("tasg"), + role, + principal, + assigned_by, + assigned_at, + }; + let assignment = match role { TicketAssignmentRole::Orchestrator => { - let store = api.store.clone(); - let notification_api = api.clone(); - let record = record.clone(); - let expected = expected_assignment_id.clone(); - let operation_id = operation_id.clone(); - api.store - .db_executor() - .run(move || { - let expected_assignment_id = expected; - let assignment = if let Some(source) = source.as_ref() { - store.set_current_ticket_orchestrator_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - source, - ) - } else { - // Only the cfg(test) direct-handler adapter lacks a Backend source. - store.set_current_ticket_role_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - ) - }?; - // Dispatch from the accepted SQL owner after commit, even - // when its waiter has gone. The response still observes the - // same best-effort notification completion as before offload. - Ok::<_, Error>(( - assignment, - enqueue_orchestrator_assignment_attention(¬ification_api), - )) - }) - .await? + if let Some(source) = source.as_ref() { + api.store.set_current_ticket_orchestrator_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + source, + )? + } else { + // Only the cfg(test) direct-handler adapter lacks a Backend source. + api.store.set_current_ticket_role_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + )? + } } TicketAssignmentRole::Worker => { if expected_assignment_id.is_some() { @@ -14636,17 +14061,14 @@ async fn apply_set_ticket_assignment( source.as_ref(), request.orchestrator_assignment_id.as_deref(), )?; - ( - start_manual_ticket_worker_assignment( - &api, - &record, - &operation_id, - &workdir_bindings, - delegation.as_ref(), - ) - .await?, - None, + start_manual_ticket_worker_assignment( + &api, + &record, + &operation_id, + &request.workdir_bindings, + delegation.as_ref(), ) + .await? } TicketAssignmentRole::Owner | TicketAssignmentRole::Contributor => { return Err(Error::TicketAssignmentConflict( @@ -14656,8 +14078,12 @@ async fn apply_set_ticket_assignment( .into()); } }; - if let Some(notification) = notification { - let _ = notification.await; + if role == TicketAssignmentRole::Orchestrator + && let Some(orchestrator) = find_workspace_orchestrator(&api) + { + // The Assign row/event/receipt has committed. Delivery failure cannot + // roll back delegation; startup/restore/idle use this same durable scan. + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); } Ok(Json(server_api::TicketRoleAssignmentMutationResponse { workspace_id, @@ -14683,13 +14109,7 @@ async fn scoped_clear_ticket_assignment_with_context( query: Query, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let source_api = api.clone(); - let context = context.clone(); - let (author, source) = api - .host_executors - .host()? - .run(move || authenticated_assignment_source(&source_api, &headers, &context)) - .await?; + let (author, source) = authenticated_assignment_source(&api, &headers, context)?; apply_clear_ticket_assignment(State(api), path, query, author, Some(source)).await } @@ -14700,70 +14120,65 @@ async fn apply_clear_ticket_assignment( author: String, source: Option, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = query - .operation_id - .map(|value| require_ticket_assignment_value("operation_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) - })?; - let assignment_id = query - .assignment_id - .map(|value| require_ticket_assignment_value("assignment_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) - })?; - let event_id = new_id("tasev"); - let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let reason = Some("role assignment removed from Ticket detail"); - let cleared = if role == TicketAssignmentRole::Orchestrator - && let Some(source) = source.as_ref() - { - api.store.clear_current_ticket_orchestrator_assignment( - &workspace_id, - &ticket.id, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - source, - )? - } else { - api.store.clear_current_ticket_role_assignment( - &workspace_id, - &ticket.id, - role, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - )? - }; - if !cleared { - return Err(Error::TicketAssignmentConflict(format!( - "assignment `{assignment_id}` is not current for role `{}`", - role.as_str() - )) - .into()); - } - Ok(Json(server_api::TicketRoleAssignmentMutationResponse { - workspace_id, - ticket_id: ticket.id, - assignment: None, - })) - }) - .await + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = query + .operation_id + .map(|value| require_ticket_assignment_value("operation_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) + })?; + let assignment_id = query + .assignment_id + .map(|value| require_ticket_assignment_value("assignment_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) + })?; + let event_id = new_id("tasev"); + let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let reason = Some("role assignment removed from Ticket detail"); + let cleared = if role == TicketAssignmentRole::Orchestrator + && let Some(source) = source.as_ref() + { + api.store.clear_current_ticket_orchestrator_assignment( + &workspace_id, + &ticket.id, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + source, + )? + } else { + api.store.clear_current_ticket_role_assignment( + &workspace_id, + &ticket.id, + role, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + )? + }; + if !cleared { + return Err(Error::TicketAssignmentConflict(format!( + "assignment `{assignment_id}` is not current for role `{}`", + role.as_str() + )) + .into()); + } + Ok(Json(server_api::TicketRoleAssignmentMutationResponse { + workspace_id, + ticket_id: ticket.id, + assignment: None, + })) } async fn scoped_cancel_ticket_implementation( @@ -15282,54 +14697,49 @@ async fn scoped_edit_ticket_item( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let mut targets = request.target.map(|target| match target { - server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { - targets: targets - .into_iter() - .map(|target| ticket::TicketTarget { - repository_key: target.repository_key, - ref_selector: target.ref_selector, - access: target.access, - }) - .collect(), + validate_workspace_scope(&api, &path.workspace_id)?; + let mut targets = request.target.map(|target| match target { + server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { + targets: targets + .into_iter() + .map(|target| ticket::TicketTarget { + repository_key: target.repository_key, + ref_selector: target.ref_selector, + access: target.access, + }) + .collect(), + }, + server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, + }); + if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { + validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; + } + browser_ticket_backend(&api)? + .edit_item( + TicketIdOrSlug::Id(path.id.clone()), + TicketItemEdit { + title: request.title, + body: request.body.map(MarkdownText::new), + body_replacement: match (request.old_string, request.new_string) { + (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { + old_string, + new_string, + replace_all: request.replace_all, + }), + (None, None) => None, + _ => { + return Err(settings_bad_request( + "invalid_ticket_edit_replacement", + "old_string and new_string must be provided together", + )); + } }, - server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, - }); - if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { - validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; - } - browser_ticket_backend(&api)? - .edit_item( - TicketIdOrSlug::Id(path.id.clone()), - TicketItemEdit { - title: request.title, - body: request.body.map(MarkdownText::new), - body_replacement: match (request.old_string, request.new_string) { - (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { - old_string, - new_string, - replace_all: request.replace_all, - }), - (None, None) => None, - _ => { - return Err(settings_bad_request( - "invalid_ticket_edit_replacement", - "old_string and new_string must be provided together", - )); - } - }, - targets, - author: request.author, - }, - ) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) - }) - .await + targets, + author: request.author, + }, + ) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) } async fn scoped_append_ticket_event( @@ -15337,26 +14747,21 @@ async fn scoped_append_ticket_event( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = match request.role { - server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, - server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, - server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, - server_api::BrowserTicketThreadRole::ImplementationReport => { - TicketEventKind::ImplementationReport - } - }; - let mut event = NewTicketEvent::new(role, request.body); - event.author = request.author; - browser_ticket_backend(&api)? - .add_event(TicketIdOrSlug::Id(path.id.clone()), event) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let role = match request.role { + server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, + server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, + server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, + server_api::BrowserTicketThreadRole::ImplementationReport => { + TicketEventKind::ImplementationReport + } + }; + let mut event = NewTicketEvent::new(role, request.body); + event.author = request.author; + browser_ticket_backend(&api)? + .add_event(TicketIdOrSlug::Id(path.id.clone()), event) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) } async fn scoped_mark_ticket_ready_from_browser( @@ -15535,24 +14940,17 @@ fn schedule_ticket_item_check( } }; let api = api.clone(); - let admission = api.host_executors.host().and_then(|executor| { - executor.dispatch(move || { - if let Err(error) = api.dispatch_backend_job_reservation(reservation) { - tracing::warn!( - job_id = %request.job_id, - ticket_input_digest = ?request.input_digest(), - %error, - "asynchronous Ticket item checker dispatch failed" - ); - if !matches!(error, Error::DbOperationOutcomeUnknown) { - api.drain_reserved_backend_jobs(); - } - } - }) + tokio::task::spawn_blocking(move || { + if let Err(error) = api.dispatch_backend_job_reservation(reservation) { + tracing::warn!( + job_id = %request.job_id, + ticket_input_digest = ?request.input_digest(), + %error, + "asynchronous Ticket item checker dispatch failed" + ); + api.drain_reserved_backend_jobs(); + } }); - if let Err(error) = admission { - tracing::warn!(%error, "Ticket checker was not admitted; its reservation remains durable"); - } } async fn execute_ticket_rest_operation( @@ -15588,13 +14986,6 @@ async fn execute_ticket_rest_operation_with_context( .await?; return Ok(TicketBackendOperationResult::Ticket(updated)); } - let job_api = api.clone(); - let workspace_id = workspace_id.to_owned(); - let context = context.cloned(); - api.host_executors.host()?.run(move || { - let api = &job_api; - let workspace_id = workspace_id.as_str(); - let context = context.as_ref(); validate_ticket_operation_repository_keys(api, workspace_id, &operation)?; let mut backend = SqliteTicketBackend::open_verified( api.config.database_path.clone(), @@ -15792,7 +15183,6 @@ async fn execute_ticket_rest_operation_with_context( } } Ok(result) - }).await } #[cfg(test)] @@ -16755,8 +16145,6 @@ async fn scoped_list_merge_requests( AxumPath(workspace_id): AxumPath, Query(query): Query, ) -> ApiResult> { - api.host_executors.host()?.run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; require_workspace_access(&workspace_id, &api)?; let ticket_id = query @@ -16865,8 +16253,6 @@ async fn scoped_list_merge_requests( items, next_cursor: page.next_cursor, })) - - }).await } fn merge_ref_diagnostic(error: ApiError) -> MergeRequestRefDiagnostic { @@ -16972,169 +16358,159 @@ async fn scoped_show_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let store = merge_request_store(&api, &workspace_id)?; - let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let integrated_refs = integrated_merge_refs(&mr); - mr.thread = store.thread_page_by_id( + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let store = merge_request_store(&api, &workspace_id)?; + let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let integrated_refs = integrated_merge_refs(&mr); + mr.thread = store.thread_page_by_id( + &workspace_id, + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + let ticket_id = mr.ticket_ids.first().ok_or_else(|| { + Error::MergeRequest(merge_request::MergeRequestError::Conflict( + "Merge Request has no linked Ticket".into(), + )) + })?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; + let (source, target) = if let Some(refs) = integrated_refs { + refs + } else { + let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { + (Some(selector), Some(assignment)) => { + match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + } + } + (Some(_), None) => unknown_merge_ref( + "source_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the source ref", + ), + (None, _) => server_api::MergeRequestRefResponse { + status: "requires_repair".into(), + resolved_ref: None, + observed_at: Utc::now().to_rfc3339(), + diagnostic: None, + }, + }; + let target = match assignment.as_ref() { + Some(assignment) => match observe_published_merge_ref( + &api, &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - let ticket_id = mr.ticket_ids.first().ok_or_else(|| { - Error::MergeRequest(merge_request::MergeRequestError::Conflict( - "Merge Request has no linked Ticket".into(), - )) - })?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; - let (source, target) = if let Some(refs) = integrated_refs { - refs - } else { - let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { - (Some(selector), Some(assignment)) => { - match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - } - } - (Some(_), None) => unknown_merge_ref( - "source_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the source ref", - ), - (None, _) => server_api::MergeRequestRefResponse { - status: "requires_repair".into(), - resolved_ref: None, - observed_at: Utc::now().to_rfc3339(), - diagnostic: None, - }, - }; - let target = match assignment.as_ref() { - Some(assignment) => match observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - }, - None => unknown_merge_ref( - "target_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the target ref", - ), - }; - (source, target) - }; - let linked_tickets = mr - .ticket_ids - .iter() - .map(|ticket_id| { - Ok(server_api::MergeRequestLinkedTicketResponse { - ticket_id: ticket_id.clone(), - key: api.store.resource_key( - &workspace_id, - WorkspaceResourceKind::Ticket, - ticket_id, - )?, - }) - }) - .collect::>>()?; - let merge_request = public_merge_request(&api, &workspace_id, mr)?; - Ok(Json(server_api::MergeRequestDetailResponse { - merge_request, - source, - target, - linked_tickets, - })) + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + }, + None => unknown_merge_ref( + "target_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the target ref", + ), + }; + (source, target) + }; + let linked_tickets = mr + .ticket_ids + .iter() + .map(|ticket_id| { + Ok(server_api::MergeRequestLinkedTicketResponse { + ticket_id: ticket_id.clone(), + key: api.store.resource_key( + &workspace_id, + WorkspaceResourceKind::Ticket, + ticket_id, + )?, + }) }) - .await + .collect::>>()?; + let merge_request = public_merge_request(&api, &workspace_id, mr)?; + Ok(Json(server_api::MergeRequestDetailResponse { + merge_request, + source, + target, + linked_tickets, + })) } async fn scoped_merge_request_readiness( State(api): State, AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; - let (current_subject_ref, source_blocker) = - if mr.state == merge_request::MergeRequestState::Merged { - ( - mr.merged_result() - .ok() - .map(|merge| merge.approved_source_ref.clone()), - None, - ) - } else { - match (mr.selector_from.as_deref(), assignment) { - (Some(selector), Some(assignment)) => match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => (Some(observation.resolved_ref), None), - Err(error) => { - let diagnostic = merge_ref_diagnostic(error); - let blocker = source_ref_readiness_blocker(&diagnostic.code); - (None, Some(blocker.to_string())) - } - }, - (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), - (None, _) => (None, None), + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; + let (current_subject_ref, source_blocker) = + if mr.state == merge_request::MergeRequestState::Merged { + ( + mr.merged_result() + .ok() + .map(|merge| merge.approved_source_ref.clone()), + None, + ) + } else { + match (mr.selector_from.as_deref(), assignment) { + (Some(selector), Some(assignment)) => match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => (Some(observation.resolved_ref), None), + Err(error) => { + let diagnostic = merge_ref_diagnostic(error); + let blocker = source_ref_readiness_blocker(&diagnostic.code); + (None, Some(blocker.to_string())) } - }; - let mut report = store.readiness(merge_request::ReadinessCheck { - merge_request_id, - ticket_id, - current_subject_ref, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: String::new(), - worker_id: String::new(), - assignment_id: String::new(), }, - })?; - if let Some(blocker) = source_blocker { - report - .blockers - .retain(|current| current != "selector_unresolved"); - if !report.blockers.contains(&blocker) { - report.blockers.push(blocker); - } - report.ready = false; + (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), + (None, _) => (None, None), } - Ok(Json(server_api::MergeRequestReadinessResponse { - ready: report.ready, - blockers: report.blockers, - subject_ref: report.subject_ref, - review: report.review.map(public_review_event), - })) - }) - .await + }; + let mut report = store.readiness(merge_request::ReadinessCheck { + merge_request_id, + ticket_id, + current_subject_ref, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: String::new(), + worker_id: String::new(), + assignment_id: String::new(), + }, + })?; + if let Some(blocker) = source_blocker { + report + .blockers + .retain(|current| current != "selector_unresolved"); + if !report.blockers.contains(&blocker) { + report.blockers.push(blocker); + } + report.ready = false; + } + Ok(Json(server_api::MergeRequestReadinessResponse { + ready: report.ready, + blockers: report.blockers, + subject_ref: report.subject_ref, + review: report.review.map(public_review_event), + })) } async fn scoped_open_merge_request( @@ -17143,79 +16519,71 @@ async fn scoped_open_merge_request( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict( - "Ticket has no active Coder work assignment".into(), - ) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let repository = - api.require_configured_workspace_repository_by_key(&input.repository_key)?; - let repository_id = repository.id; - require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_from, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &repository_id, - &input.selector_from, - &source_observation.resolved_ref, - )?; - observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_to, - )?; - let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( - merge_request::OpenMergeRequest { - merge_request_id: Uuid::now_v7().to_string(), - ticket_id, - repository_id: repository_id.clone(), - selector_from: input.selector_from, - selector_to: input.selector_to, - summary: input.summary, - auth: merge_request::MergeRequestAuth { - workspace_id: workspace_id.clone(), - repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }, - )?; - Ok(Json(public_merge_request( - &api, - &workspace_id, - merge_request, - )?)) - }) - .await + let workspace_id = parse_workspace_id(&workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let repository = api.require_configured_workspace_repository_by_key(&input.repository_key)?; + let repository_id = repository.id; + require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_from, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &repository_id, + &input.selector_from, + &source_observation.resolved_ref, + )?; + observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_to, + )?; + let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( + merge_request::OpenMergeRequest { + merge_request_id: Uuid::now_v7().to_string(), + ticket_id, + repository_id: repository_id.clone(), + selector_from: input.selector_from, + selector_to: input.selector_to, + summary: input.summary, + auth: merge_request::MergeRequestAuth { + workspace_id: workspace_id.clone(), + repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }, + )?; + Ok(Json(public_merge_request( + &api, + &workspace_id, + merge_request, + )?)) } async fn scoped_merge_request_thread( @@ -17223,24 +16591,19 @@ async fn scoped_merge_request_thread( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult>> { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( - &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - Ok(Json( - events - .into_iter() - .map(public_merge_request_thread_event) - .collect(), - )) - }) - .await + let workspace_id = parse_workspace_id(&workspace_id)?; + let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( + &workspace_id, + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + Ok(Json( + events + .into_iter() + .map(public_merge_request_thread_event) + .collect(), + )) } async fn scoped_repair_merge_request_selector( @@ -17295,25 +16658,20 @@ async fn scoped_register_reviewer_child_session( AxumPath(workspace_id): AxumPath, Json(input): Json, ) -> ApiResult { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( - merge_request::RegisterReviewerChildSession { - workspace_id, - parent_runtime_id: source.runtime_id, - parent_worker_id: source.worker_id, - child_session_id: input.child_session_id, - reviewer_profile: "builtin:reviewer".into(), - now: Utc::now(), - }, - )?; - Ok(StatusCode::NO_CONTENT) - }) - .await + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( + merge_request::RegisterReviewerChildSession { + workspace_id, + parent_runtime_id: source.runtime_id, + parent_worker_id: source.worker_id, + child_session_id: input.child_session_id, + reviewer_profile: "builtin:reviewer".into(), + now: Utc::now(), + }, + )?; + Ok(StatusCode::NO_CONTENT) } async fn scoped_register_merge_request_review_capability( @@ -17322,115 +16680,107 @@ async fn scoped_register_merge_request_review_capability( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let requested_ticket_id = - resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; - if requested_ticket_id != ticket_id { - return Err(Error::TicketAssignmentConflict( - "Reviewer handoff Ticket does not match the addressed Merge Request".into(), - ) - .into()); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict( - "Ticket has no active Coder work assignment".into(), - ) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr.selector_from.as_deref().ok_or_else(|| { - Error::InvalidInput("selector_from requires repair".into()) - })?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &mr.repository_id, - selector, - &source_observation.resolved_ref, - )?; - source_observation.resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err(Error::InvalidInput( - "closed Merge Request is not reviewable".into(), - ) - .into()); - } - }; - let mut ticket_merge_request_subjects = Vec::new(); - for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { - let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged - { - merged_result_source_ref(&linked)? - } else { - let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { - Error::InvalidInput(format!( - "Merge Request `{}` selector_from requires repair", - linked.merge_request_id - )) - })?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &linked.repository_id, - linked_selector, - )? - .resolved_ref - }; - ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { - merge_request_id: linked.merge_request_id, - subject_ref: linked_subject_ref, - }); - } - let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; - store.request_review(merge_request::RequestMergeRequestReview { - merge_request_id, - ticket_id, - ticket_content_digest, - ticket_merge_request_subjects, - subject_ref, - child_session_id: input.child_session_id, - capability_token: input.capability_token, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let requested_ticket_id = + resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; + if requested_ticket_id != ticket_id { + return Err(Error::TicketAssignmentConflict( + "Reviewer handoff Ticket does not match the addressed Merge Request".into(), + ) + .into()); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr + .selector_from + .as_deref() + .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &mr.repository_id, + selector, + &source_observation.resolved_ref, + )?; + source_observation.resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err( + Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), + ); + } + }; + let mut ticket_merge_request_subjects = Vec::new(); + for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { + let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged { + merged_result_source_ref(&linked)? + } else { + let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput(format!( + "Merge Request `{}` selector_from requires repair", + linked.merge_request_id + )) })?; - Ok(StatusCode::NO_CONTENT) - }) - .await + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &linked.repository_id, + linked_selector, + )? + .resolved_ref + }; + ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { + merge_request_id: linked.merge_request_id, + subject_ref: linked_subject_ref, + }); + } + let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; + store.request_review(merge_request::RequestMergeRequestReview { + merge_request_id, + ticket_id, + ticket_content_digest, + ticket_merge_request_subjects, + subject_ref, + child_session_id: input.child_session_id, + capability_token: input.capability_token, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + })?; + Ok(StatusCode::NO_CONTENT) } async fn scoped_submit_merge_request_review( @@ -17438,74 +16788,67 @@ async fn scoped_submit_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let review_authorization = - store.authorize_review_submission(&merge_request_id, &input.capability_token)?; - if review_authorization.workspace_id != workspace_id - || review_authorization.merge_request_id != merge_request_id - { - return Err( - Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( - "review grant invalid".into(), - )) - .into(), - ); - } - let ticket_id = review_authorization.ticket_id; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict( - "Ticket has no active Coder work assignment".into(), - ) - })?; - let current_subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr.selector_from.as_deref().ok_or_else(|| { - Error::InvalidInput("selector_from requires repair".into()) - })?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err(Error::InvalidInput( - "closed Merge Request is not reviewable".into(), - ) - .into()); - } - }; - let decision = internal_review_decision(input.decision); - let findings = input - .findings - .into_iter() - .map(internal_review_finding) - .collect(); - let event = store.submit_review(merge_request::SubmitMergeRequestReview { - merge_request_id, - ticket_id, - current_subject_ref, - capability_token: input.capability_token, - decision, - body: input.body, - findings, - now: Utc::now(), - })?; - Ok(Json(public_review_event(event))) - }) - .await + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let review_authorization = + store.authorize_review_submission(&merge_request_id, &input.capability_token)?; + if review_authorization.workspace_id != workspace_id + || review_authorization.merge_request_id != merge_request_id + { + return Err( + Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( + "review grant invalid".into(), + )) + .into(), + ); + } + let ticket_id = review_authorization.ticket_id; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) + })?; + let current_subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr + .selector_from + .as_deref() + .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err( + Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), + ); + } + }; + let decision = internal_review_decision(input.decision); + let findings = input + .findings + .into_iter() + .map(internal_review_finding) + .collect(); + let event = store.submit_review(merge_request::SubmitMergeRequestReview { + merge_request_id, + ticket_id, + current_subject_ref, + capability_token: input.capability_token, + decision, + body: input.body, + findings, + now: Utc::now(), + })?; + Ok(Json(public_review_event(event))) } async fn scoped_revoke_merge_request_review( @@ -17514,51 +16857,44 @@ async fn scoped_revoke_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - if !input.explicit_confirmation { - return Err(Error::BrowserReopenConfirmationRequired.into()); - } - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict( - "Ticket has no active Coder work assignment".into(), - ) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let event = store.revoke_review(merge_request::RevokeMergeRequestReview { - merge_request_id, - ticket_id, - review_event_id: input.review_event_id, - reason: input.reason, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - })?; - Ok(Json(public_review_revoked_event(event))) - }) - .await + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + if !input.explicit_confirmation { + return Err(Error::BrowserReopenConfirmationRequired.into()); + } + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let event = store.revoke_review(merge_request::RevokeMergeRequestReview { + merge_request_id, + ticket_id, + review_event_id: input.review_event_id, + reason: input.reason, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + })?; + Ok(Json(public_review_revoked_event(event))) } async fn scoped_complete_merge_request( @@ -17567,101 +16903,94 @@ async fn scoped_complete_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - require_online_workspace_orchestrator_source(&api, &source)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { - let replay = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: existing.approved_source_ref.clone(), - target_ref_before: input.target_ref_before, - target_ref_after: input.target_ref_after, - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: String::new(), - }, - now: Utc::now(), - }; - return store - .complete(replay) - .map(public_merge_event) - .map(Json) - .map_err(Into::into); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict( - "Ticket has no active Coder work assignment".into(), - ) - })?; - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - let current_source_ref = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref; - let observed_target = observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - )?; - require_completed_target_observation( - &observed_target.resolved_ref, - &input.target_ref_before, - &input.target_ref_after, - )?; - let completion = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: current_source_ref, - target_ref_before: input.target_ref_before.clone(), - target_ref_after: input.target_ref_after.clone(), - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }; - store.validate_completion(&completion)?; - store - .complete(completion) - .map(public_merge_event) - .map(Json) - .map_err(Into::into) - }) - .await + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + require_online_workspace_orchestrator_source(&api, &source)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { + let replay = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: existing.approved_source_ref.clone(), + target_ref_before: input.target_ref_before, + target_ref_after: input.target_ref_after, + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: String::new(), + }, + now: Utc::now(), + }; + return store + .complete(replay) + .map(public_merge_event) + .map(Json) + .map_err(Into::into); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) + })?; + let selector = mr + .selector_from + .as_deref() + .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; + let current_source_ref = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref; + let observed_target = observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + )?; + require_completed_target_observation( + &observed_target.resolved_ref, + &input.target_ref_before, + &input.target_ref_after, + )?; + let completion = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: current_source_ref, + target_ref_before: input.target_ref_before.clone(), + target_ref_after: input.target_ref_after.clone(), + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }; + store.validate_completion(&completion)?; + store + .complete(completion) + .map(public_merge_event) + .map(Json) + .map_err(Into::into) } // Header-only entry point is retained solely for explicit legacy internal tests. @@ -17684,56 +17013,47 @@ async fn scoped_ticket_assignment_tracking_update_with_context( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(update): Json, ) -> ApiResult> { - let context = context.clone(); - api.store - .db_executor() - .run(move || { - let context = &context; - - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let source = - authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { - Error::WorkspacePermissionDenied( - "Tracking is self-reported by a verified Runtime Worker".into(), - ) - })?; - // Tracking authority is retained responsibility plus the registered singleton, - // NOT Ticket openness, active work, Worker state or a model-supplied role. - if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { - return Err(Error::WorkspacePermissionDenied( - "Tracking updates require the registered current Workspace Orchestrator".into(), - ) - .into()); - } - let authentication_binding_id = context - .runtime_source - .as_ref() - .map(|source| source.authentication_binding_id.clone()) - .ok_or_else(|| { - Error::WorkspacePermissionDenied( + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let source = authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Tracking is self-reported by a verified Runtime Worker".into(), + ) + })?; + // Tracking authority is retained responsibility plus the registered singleton, + // NOT Ticket openness, active work, Worker state or a model-supplied role. + if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { + return Err(Error::WorkspacePermissionDenied( + "Tracking updates require the registered current Workspace Orchestrator".into(), + ) + .into()); + } + let authentication_binding_id = context + .runtime_source + .as_ref() + .map(|source| source.authentication_binding_id.clone()) + .ok_or_else(|| { + Error::WorkspacePermissionDenied( "Tracking requires the Runtime binding actually used to authenticate the request" .into(), ) - })?; - let source_context = ticket::TicketAssignmentTrackingSourceContext { - actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - agent_key: "workspace-orchestrator".into(), - authentication_binding_id, - }; - // The Ticket-owned mutation checks current assignment/CAS and exact replay in - // one transaction. A historical receipt never restores operational grants. - let value = api.store.update_ticket_assignment_tracking( - &workspace_id, - &ticket_id, - &update, - &source_context, - )?; - Ok(Json(value)) - }) - .await + })?; + let source_context = ticket::TicketAssignmentTrackingSourceContext { + actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + agent_key: "workspace-orchestrator".into(), + authentication_binding_id, + }; + // The Ticket-owned mutation checks current assignment/CAS and exact replay in + // one transaction. A historical receipt never restores operational grants. + let value = api.store.update_ticket_assignment_tracking( + &workspace_id, + &ticket_id, + &update, + &source_context, + )?; + Ok(Json(value)) } async fn scoped_ticket_assignment_tracking_history( @@ -17741,31 +17061,25 @@ async fn scoped_ticket_assignment_tracking_history( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let limit = query.limit.unwrap_or(50); - if !(1..=100).contains(&limit) { - return Err(Error::InvalidInput( - "tracking history limit must be between 1 and 100".into(), - ) - .into()); - } - let page = api.store.page_ticket_assignment_tracking_history( - &workspace_id, - &ticket_id, - &query.assignment_id, - query.after_event_id.as_deref(), - limit, - )?; - Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { - items: page.items, - next_event_id: page.next_event_id, - })) - }) - .await + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let limit = query.limit.unwrap_or(50); + if !(1..=100).contains(&limit) { + return Err( + Error::InvalidInput("tracking history limit must be between 1 and 100".into()).into(), + ); + } + let page = api.store.page_ticket_assignment_tracking_history( + &workspace_id, + &ticket_id, + &query.assignment_id, + query.after_event_id.as_deref(), + limit, + )?; + Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { + items: page.items, + next_event_id: page.next_event_id, + })) } async fn scoped_ticket_case_update_with_context( @@ -17834,10 +17148,6 @@ async fn apply_ticket_case_update( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(mut request): Json, ) -> ApiResult> { - let actor = actor.cloned(); - api.store.db_executor().run(move || { - let actor = actor.as_ref(); - validate_workspace_scope(&api, &workspace_id)?; let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; let mut backend = browser_ticket_backend(&api)?; @@ -17894,8 +17204,6 @@ async fn apply_ticket_case_update( .update_case(&ticket_id, request) .map_err(Error::from)?; Ok(Json(ticket)) - - }).await } fn reject_non_browser_reopen_auth(headers: &HeaderMap) -> Result<()> { @@ -18463,93 +17771,44 @@ fn runtime_local_owner_worker_id<'a>( (caller.runtime_id == target_runtime_id).then_some(caller.worker_id.as_str()) } -// Locked async workflows stay on Tokio. Each accepted synchronous mutation -// retains this same owned guard through execution and discarded-result cleanup. -async fn run_worker_workdir_db_stage_locked( - api: &WorkspaceApi, - session_guard: &Arc>, - operation: impl FnOnce(WorkspaceApi) -> std::result::Result + Send + 'static, -) -> std::result::Result -where - T: Send + 'static, - E: From + Send + 'static, -{ - let stage_api = api.clone(); - let stage_guard = session_guard.clone(); - let (outcome, _session_guard) = api - .store - .db_executor() - .run(move || { - let outcome = operation(stage_api); - // Keep the guard with the outcome, including errors. If the waiter was - // cancelled, the job drops the outcome before releasing this guard. - Ok::<_, E>((outcome, stage_guard)) - }) - .await?; - outcome -} - async fn open_current_worker_workdir_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, link: &WorkerWorkdirLinkRecord, - session_guard: &Arc>, ) -> Result { - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let stage_link = link.clone(); - let (workdir, effective_capabilities) = api + let workdir = api .store - .db_executor() - .run(move || { - let workdir = stage_api - .store - .list_workdir_registry(&stage_api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == stage_link.workdir_id) - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: stage_worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!( - "attached Workdir {} is not registered in this Workspace", - stage_link.workdir_id - ), - })?; - if matches!( - workdir.source, - WorkdirRegistrySource::WorkspaceConfig { .. } - ) { - return Err(Error::WorkspacePermissionDenied( - "Logical config has no filesystem session".into(), - )); - } - let capabilities = effective_worker_workdir_capabilities( - &stage_api, - &stage_worker, - &workdir, - stage_link.capabilities, - )?; - Ok((workdir, capabilities)) - }) - .await?; + .list_workdir_registry(&api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == link.workdir_id) + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!( + "attached Workdir {} is not registered in this Workspace", + link.workdir_id + ), + })?; + if matches!( + workdir.source, + WorkdirRegistrySource::WorkspaceConfig { .. } + ) { + return Err(Error::WorkspacePermissionDenied( + "Logical config has no filesystem session".into(), + )); + } + let effective_capabilities = + effective_worker_workdir_capabilities(api, worker, &workdir, link.capabilities)?; if let WorkdirRegistrySource::ExternalGrant { grant_id } = &workdir.source { close_current_worker_attachment_session_locked(api, worker).await?; - let stage_api = api.clone(); - let stage_grant_id = grant_id.clone(); let mut grant = api .store - .db_executor() - .run(move || { - stage_api - .store - .get_external_workdir_grant(&stage_api.config.workspace_id, &stage_grant_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "external_workdir_grant_not_found".to_string(), - message: "External Workdir grant is unavailable".to_string(), - }) - }) - .await?; + .get_external_workdir_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "external_workdir_grant_not_found".to_string(), + message: "External Workdir grant is unavailable".to_string(), + })?; let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(api, grant_id) @@ -18557,13 +17816,9 @@ async fn open_current_worker_workdir_session_locked( .map_err(|error| error.error)?; } if matches!(grant.status.as_str(), "revoked" | "expired") { - cleanup_external_workdir_attachments_with_lock( - api, - &grant.workdir_id, - Some((worker, session_guard)), - ) - .await - .map_err(|error| error.error)?; + cleanup_external_workdir_attachments_with_lock(api, &grant.workdir_id, Some(worker)) + .await + .map_err(|error| error.error)?; cancel_external_workdir_expiry(api, grant_id); } if grant.status != "online" { @@ -18630,63 +17885,31 @@ async fn open_current_worker_workdir_session_locked( }); } }; - let stage_api = api.clone(); - let stage_repository_id = repository_id.to_string(); let repository = api - .store - .db_executor() - .run(move || { - stage_api - .require_configured_workspace_repository(&stage_repository_id) - .map_err(|error| error.error) - }) - .await?; + .require_configured_workspace_repository(repository_id) + .map_err(|error| error.error)?; let repository_requires_access = repository.source.kind == server_api::RepositorySourceKind::Ssh; if repository_requires_access { close_current_worker_attachment_session_locked(api, worker).await?; - let stage_api = api.clone(); - let stage_runtime_id = runtime_id.to_string(); - let stage_workdir_id = workdir.workdir_id.clone(); - let operation_id = format!( - "workdir-session:{}:{}:{}", - worker.runtime_id, worker.worker_id, workdir.workdir_id - ); - let stage_guard = session_guard.clone(); - let access = api - .store - .db_executor() - .run(move || { - let _session_guard = stage_guard; - repository_access_request_for_workdir( - &stage_api, - &stage_runtime_id, - &stage_workdir_id, - &operation_id, - ) - .map_err(|error| error.error)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: stage_runtime_id, - code: "working_directory_remote_repository_access_required".to_string(), - message: "current Repository SSH access authority is unavailable".to_string(), - }) - }) - .await?; - let stage_api = api.clone(); - let stage_runtime_id = runtime_id.to_string(); - // The accepted Runtime authorization must retain the actual session lock, - // not just the async waiter that submitted it. - let stage_guard = session_guard.clone(); - api.host_executors - .host()? - .run(move || { - let _session_guard = stage_guard; - stage_api - .runtime - .authorize_working_directory_repository_access(&stage_runtime_id, access) - .map_err(RuntimeRegistryError::into_error) - }) - .await?; + let access = repository_access_request_for_workdir( + api, + runtime_id, + &workdir.workdir_id, + &format!( + "workdir-session:{}:{}:{}", + worker.runtime_id, worker.worker_id, workdir.workdir_id + ), + ) + .map_err(|error| error.error)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: runtime_id.to_string(), + code: "working_directory_remote_repository_access_required".to_string(), + message: "current Repository SSH access authority is unavailable".to_string(), + })?; + api.runtime + .authorize_working_directory_repository_access(runtime_id, access) + .map_err(RuntimeRegistryError::into_error)?; } else { // The registry is an operation-session cache, not routing authority. // Close any prior alias session before opening the explicitly selected @@ -18815,10 +18038,10 @@ fn restore_logical_workdir_attachments( .map_err(ApiError::from) } -fn prepare_runtime_worker_workdir_attachments( +fn sync_runtime_worker_workdir_attachments( api: &WorkspaceApi, worker: &RuntimeWorkerRef, -) -> ApiResult> { +) -> ApiResult<()> { let attachments = api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? @@ -18861,14 +18084,6 @@ fn prepare_runtime_worker_workdir_attachments( }) }) .collect::>>()?; - Ok(attachments) -} - -fn replace_runtime_worker_workdir_attachments( - api: &WorkspaceApi, - worker: &RuntimeWorkerRef, - attachments: Vec, -) -> ApiResult<()> { let result = api .runtime .replace_worker_workdir_attachments(worker, attachments) @@ -18888,68 +18103,12 @@ fn replace_runtime_worker_workdir_attachments( Ok(()) } -#[cfg(test)] -fn sync_runtime_worker_workdir_attachments( - api: &WorkspaceApi, - worker: &RuntimeWorkerRef, -) -> ApiResult<()> { - let attachments = prepare_runtime_worker_workdir_attachments(api, worker)?; - replace_runtime_worker_workdir_attachments(api, worker, attachments) -} - -async fn sync_runtime_worker_workdir_attachments_locked( - api: &WorkspaceApi, - worker: &RuntimeWorkerRef, - session_guard: &Arc>, -) -> ApiResult<()> { - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let stage_guard = session_guard.clone(); - let attachments = api - .store - .db_executor() - .run(move || { - let _session_guard = stage_guard; - prepare_runtime_worker_workdir_attachments(&stage_api, &stage_worker) - }) - .await?; - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let stage_guard = session_guard.clone(); - api.host_executors - .host()? - .run(move || { - let _session_guard = stage_guard; - replace_runtime_worker_workdir_attachments(&stage_api, &stage_worker, attachments) - }) - .await -} - async fn refresh_current_worker_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, - session_guard: &Arc>, ) -> Result<()> { close_current_worker_attachment_session_locked(api, worker).await?; - let stage_api = api.clone(); - let stage_worker = worker.clone(); let remaining = api - .store - .db_executor() - .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) - .await?; - if remaining.len() == 1 { - open_current_worker_workdir_session_locked(api, worker, &remaining[0], session_guard) - .await?; - } - Ok(()) -} - -fn active_filesystem_workdir_links( - api: &WorkspaceApi, - worker: &RuntimeWorkerRef, -) -> Result> { - Ok(api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() @@ -18963,7 +18122,11 @@ fn active_filesystem_workdir_links( !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) }) }) - .collect()) + .collect::>(); + if remaining.len() == 1 { + open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await?; + } + Ok(()) } #[derive(Debug, Deserialize)] @@ -18978,107 +18141,35 @@ async fn scoped_submit_backend_job_result( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; - let binding = api - .store - .worker_registry_projection(&api.config.workspace_id, &worker)? - .and_then(|projection| projection.job) - .ok_or_else(|| { - Error::InvalidInput( - "current Worker is not bound to an active Backend Job attempt".to_string(), - ) - })?; - let attempt = api - .store - .get_backend_job_attempt( - &api.config.workspace_id, - &binding.job_id, - &binding.attempt_id, - )? - .ok_or_else(|| { - Error::Store("Backend Job attempt binding disappeared".to_string()) - })?; - let acceptance = api.accept_backend_job_result( - &worker, - &BackendJobResultSubmission { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - input_digest: attempt.input_digest, - result: request.result, - }, - )?; - Ok(Json(acceptance)) - }) - .await -} - -async fn detach_worker_workdir_stage_locked( - api: &WorkspaceApi, - worker: &RuntimeWorkerRef, - workdir_id: &str, - session_guard: &Arc>, -) -> Result> { - let worker = worker.clone(); - let workdir_id = workdir_id.to_string(); - run_worker_workdir_db_stage_locked(api, session_guard, move |api| { - api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(&workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ) - }) - .await -} - -async fn import_current_worker_workdir_observation( - api: &WorkspaceApi, - worker: &RuntimeWorkerRef, - session_guard: Option<&Arc>>, -) -> ApiResult<()> { - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let has_history = api + validate_workspace_scope(&api, &path.workspace_id)?; + let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let binding = api .store - .db_executor() - .run(move || { - stage_api - .store - .worker_workdir_link_history_exists(&stage_api.config.workspace_id, &stage_worker) - }) - .await?; - if !has_history { - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let stage_guard = session_guard.cloned(); - let observed_worker = api - .host_executors - .host()? - .run(move || { - let _session_guard = stage_guard; - stage_api - .runtime - .worker(&stage_worker) - .map_err(RuntimeRegistryError::into_error) - }) - .await?; - if !observed_worker.workdir_attachments.is_empty() { - let stage_api = api.clone(); - let stage_guard = session_guard.cloned(); - api.store - .db_executor() - .run(move || { - let _session_guard = stage_guard; - sync_worker_observation(&stage_api, &observed_worker) - }) - .await?; - } - } - Ok(()) + .worker_registry_projection(&api.config.workspace_id, &worker)? + .and_then(|projection| projection.job) + .ok_or_else(|| { + Error::InvalidInput( + "current Worker is not bound to an active Backend Job attempt".to_string(), + ) + })?; + let attempt = api + .store + .get_backend_job_attempt( + &api.config.workspace_id, + &binding.job_id, + &binding.attempt_id, + )? + .ok_or_else(|| Error::Store("Backend Job attempt binding disappeared".to_string()))?; + let acceptance = api.accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + input_digest: attempt.input_digest, + result: request.result, + }, + )?; + Ok(Json(acceptance)) } async fn scoped_attach_current_worker_workdir( @@ -19087,141 +18178,115 @@ async fn scoped_attach_current_worker_workdir( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - let stage_api = api.clone(); - let worker = api - .host_executors - .host()? - .run(move || -> ApiResult<_> { - validate_workspace_scope(&stage_api, &path.workspace_id)?; - Ok(current_worker_identity( - &stage_api, - &path.workspace_id, - &headers, - )?) - }) - .await?; + validate_workspace_scope(&api, &path.workspace_id)?; + let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; let alias = workdir::WorkdirAttachmentAlias::new(request.alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; let workdir_id = request.working_directory_id.trim(); if workdir_id.is_empty() || workdir_id.chars().any(char::is_control) { return Err(Error::InvalidRecordId(request.working_directory_id).into()); } - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let stage_workdir_id = workdir_id.to_string(); - let capabilities = api + let workdir = api .store - .db_executor() - .run(move || -> ApiResult<_> { - let api = stage_api; - let worker = stage_worker; - let workdir_id = stage_workdir_id.as_str(); - let workdir = api + .list_workdir_registry(&api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == workdir_id) + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!("unknown Workdir `{workdir_id}`"), + })?; + // A failed removal must not prevent its owner from attaching a still usable + // repository checkout to resolve current blockers. Runtime binding validates + // the actual checkout; durable pending-removal reservations still fence this + // mutation. Other sources and missing/corrupted materializations stay denied. + let repository_retry = workdir.materialization_status == "pending" + && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); + if workdir.materialization_status != "present" && !repository_retry { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir `{workdir_id}` is not available for attachment ({})", + workdir.materialization_status + )) + .into()); + } + match &workdir.source { + WorkdirRegistrySource::Repository { .. } => {} + WorkdirRegistrySource::WorkspaceConfig { .. } => { + return Err(Error::WorkspacePermissionDenied( + "Use the grant-bound Workspace config attachment operation".into(), + ) + .into()); + } + WorkdirRegistrySource::ExternalGrant { grant_id } => { + let grant = api .store - .list_workdir_registry(&api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == workdir_id) + .get_external_workdir_grant(&api.config.workspace_id, grant_id)? .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!("unknown Workdir `{workdir_id}`"), + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir grant is unavailable".to_string(), })?; - // A failed removal must not prevent its owner from attaching a still usable - // repository checkout to resolve current blockers. Runtime binding validates - // the actual checkout; durable pending-removal reservations still fence this - // mutation. Other sources and missing/corrupted materializations stay denied. - let repository_retry = workdir.materialization_status == "pending" - && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); - if workdir.materialization_status != "present" && !repository_retry { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir `{workdir_id}` is not available for attachment ({})", - workdir.materialization_status - )) + let grant_expired = external_workdir_grant_is_expired(&grant)?; + let online = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connection(grant_id) + .is_some_and(|connection| { + connection.generation == grant.generation + && connection.provider_instance_id == grant.provider_instance_id + && connection.workdir_id == workdir.workdir_id + && connection + .expires_at + .is_none_or(|expires_at| expires_at > Utc::now()) + }); + if grant.status != "online" || grant_expired || !online { + return Err(Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir provider is offline or expired".to_string(), + } .into()); } - match &workdir.source { - WorkdirRegistrySource::Repository { .. } => {} - WorkdirRegistrySource::WorkspaceConfig { .. } => { - return Err(Error::WorkspacePermissionDenied( - "Use the grant-bound Workspace config attachment operation".into(), - ) - .into()); - } - WorkdirRegistrySource::ExternalGrant { grant_id } => { - let grant = api - .store - .get_external_workdir_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir grant is unavailable".to_string(), - })?; - let grant_expired = external_workdir_grant_is_expired(&grant)?; - let online = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connection(grant_id) - .is_some_and(|connection| { - connection.generation == grant.generation - && connection.provider_instance_id == grant.provider_instance_id - && connection.workdir_id == workdir.workdir_id - && connection - .expires_at - .is_none_or(|expires_at| expires_at > Utc::now()) - }); - if grant.status != "online" || grant_expired || !online { - return Err(Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir provider is offline or expired".to_string(), - } - .into()); - } - } - } - Ok(ticket_target_capabilities_for_workdir( - &api, &worker, &workdir, - )?) - }) - .await?; + } + } + let capabilities = ticket_target_capabilities_for_workdir(&api, &worker, &workdir)?; let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = Arc::new(session_lock.lock_owned().await); - import_current_worker_workdir_observation(&api, &worker, Some(&_session_guard)).await?; - let stage_worker = worker.clone(); - let stage_alias = alias.to_string(); - let stage_workdir_id = workdir_id.to_string(); - let existing = - run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| -> ApiResult<_> { - let existing = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &stage_worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias); - if let Some(existing) = existing { - if existing.workdir_id != stage_workdir_id { - return Err(Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} attachment alias `{stage_alias}` is already bound to Workdir {}", - stage_worker.runtime_id, stage_worker.worker_id, existing.workdir_id - )).into()); - } - let effective_capabilities = existing.capabilities.intersection(capabilities); - if effective_capabilities != existing.capabilities { - let mut attenuated = existing; - attenuated.capabilities = effective_capabilities; - api.store.attach_worker_workdir(&attenuated)?; - } - Ok(Some(effective_capabilities)) - } else { - Ok(None) - } - }) - .await?; - if let Some(effective_capabilities) = existing { - sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; + let _session_guard = session_lock.lock().await; + if !api + .store + .worker_workdir_link_history_exists(&api.config.workspace_id, &worker)? + { + let observed_worker = api + .runtime + .worker(&worker) + .map_err(|error| error.into_error())?; + if !observed_worker.workdir_attachments.is_empty() { + sync_worker_observation(&api, &observed_worker)?; + } + } + if let Some(existing) = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == alias.as_str()) + { + if existing.workdir_id != workdir_id { + return Err(Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} attachment alias `{alias}` is already bound to Workdir {}", + worker.runtime_id, worker.worker_id, existing.workdir_id + )) + .into()); + } + let effective_capabilities = existing.capabilities.intersection(capabilities); + if effective_capabilities != existing.capabilities { + let mut attenuated = existing.clone(); + attenuated.capabilities = effective_capabilities; + api.store.attach_worker_workdir(&attenuated)?; + } + sync_runtime_worker_workdir_attachments(&api, &worker)?; api.worker_projection - .refresh_async(&worker) - .await + .refresh(&worker) .map_err(ApiError::from)?; return Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { workspace_id: api.config.workspace_id.clone(), @@ -19232,7 +18297,7 @@ async fn scoped_attach_current_worker_workdir( })); } close_current_worker_attachment_session_locked(&api, &worker).await?; - let new_link = WorkerWorkdirLinkRecord { + let link = api.store.attach_worker_workdir(&WorkerWorkdirLinkRecord { connection_id: String::new(), workspace_id: api.config.workspace_id.clone(), worker: worker.clone(), @@ -19241,46 +18306,56 @@ async fn scoped_attach_current_worker_workdir( capabilities, linked_at: Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), unlinked_at: None, - }; - let link = run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| { - api.store.attach_worker_workdir(&new_link) - }) - .await?; - if let Err(error) = - sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await - { - let _ = - detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; + })?; + if let Err(error) = sync_runtime_worker_workdir_attachments(&api, &worker) { + let _ = api.store.detach_worker_workdir( + &api.config.workspace_id, + &worker, + Some(workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ); return Err(error); } - let stage_api = api.clone(); - let stage_worker = worker.clone(); let active_count = api .store - .db_executor() - .run(move || { - active_filesystem_workdir_links(&stage_api, &stage_worker).map(|links| links.len()) + .list_worker_workdir_links(&api.config.workspace_id, &worker)? + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) }) - .await?; + .count(); if active_count == 1 - && let Err(error) = - open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await + && let Err(error) = open_current_worker_workdir_session_locked(&api, &worker, &link).await { - let _ = - detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; - let _ = - sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await; + let _ = api.store.detach_worker_workdir( + &api.config.workspace_id, + &worker, + Some(workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ); + let _ = sync_runtime_worker_workdir_attachments(&api, &worker); return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh_async(&worker) - .await + .refresh(&worker) .map_err(ApiError::from) { - detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await?; - sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; - refresh_current_worker_session_locked(&api, &worker, &_session_guard).await?; + api.store.detach_worker_workdir( + &api.config.workspace_id, + &worker, + Some(workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + )?; + sync_runtime_worker_workdir_attachments(&api, &worker)?; + refresh_current_worker_session_locked(&api, &worker).await?; return Err(error); } Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -19410,12 +18485,7 @@ async fn scoped_detach_current_worker_workdir( validate_workspace_scope(&api, &workspace_id)?; let alias = workdir::WorkdirAttachmentAlias::new(alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; - let stage_api = api.clone(); - let worker = api - .host_executors - .host()? - .run(move || current_worker_identity(&stage_api, &workspace_id, &headers)) - .await?; + let worker = current_worker_identity(&api, &workspace_id, &headers)?; detach_current_worker_workdir( &api, &worker, @@ -19426,18 +18496,6 @@ async fn scoped_detach_current_worker_workdir( .map(Json) } -async fn restore_worker_workdir_connection_stage_locked( - api: &WorkspaceApi, - link: &WorkerWorkdirLinkRecord, - session_guard: &Arc>, -) -> Result<()> { - let link = link.clone(); - run_worker_workdir_db_stage_locked(api, session_guard, move |api| { - api.store.restore_worker_workdir_connection(&link) - }) - .await -} - async fn detach_current_worker_workdir( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -19445,84 +18503,70 @@ async fn detach_current_worker_workdir( expected_connection_id: Option<&str>, ) -> ApiResult { let session_lock = current_worker_session_lock(api, worker); - let _session_guard = Arc::new(session_lock.lock_owned().await); - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let stage_alias = alias.to_string(); - let expected_connection_id = expected_connection_id.map(str::to_string); + let _session_guard = session_lock.lock().await; let link = api .store - .db_executor() - .run(move || -> ApiResult<_> { - let link = stage_api - .store - .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias) - .ok_or_else(|| { - Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} has no Workdir attachment `{stage_alias}`", - stage_worker.runtime_id, stage_worker.worker_id - )) - })?; - if expected_connection_id - .as_ref() - .is_some_and(|expected| *expected != link.connection_id) - { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir attachment `{stage_alias}` connection changed during detach" - )) - .into()); - } - Ok(link) - }) - .await?; + .list_worker_workdir_links(&api.config.workspace_id, worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == alias) + .ok_or_else(|| { + Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} has no Workdir attachment `{alias}`", + worker.runtime_id, worker.worker_id + )) + })?; + if expected_connection_id.is_some_and(|expected| expected != link.connection_id) { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir attachment `{alias}` connection changed during detach" + )) + .into()); + } close_current_worker_attachment_session_locked(api, worker).await?; close_current_worker_command_sessions_for_alias_locked(api, worker, alias).await?; // Even unconditional HTTP requests condition the mutation on the lifetime just read // under the session lock, so a concurrent ledger replacement cannot be removed. - let stage_link = link.clone(); - run_worker_workdir_db_stage_locked(api, &_session_guard, move |api| { - api.store.detach_worker_workdir_connection( - &api.config.workspace_id, - &stage_link.worker, - &stage_link.alias, - &stage_link.connection_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ) - }) - .await?; - if let Err(error) = - sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await - { - restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; + api.store.detach_worker_workdir_connection( + &api.config.workspace_id, + worker, + alias, + &link.connection_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + )?; + if let Err(error) = sync_runtime_worker_workdir_attachments(api, worker) { + api.store.restore_worker_workdir_connection(&link)?; return Err(error); } - let stage_api = api.clone(); - let stage_worker = worker.clone(); let remaining = api .store - .db_executor() - .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) - .await?; + .list_worker_workdir_links(&api.config.workspace_id, worker)? + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + }) + .collect::>(); if remaining.len() == 1 && let Err(error) = - open_current_worker_workdir_session_locked(api, worker, &remaining[0], &_session_guard) - .await + open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await { - restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; - let _ = sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await; + api.store.restore_worker_workdir_connection(&link)?; + let _ = sync_runtime_worker_workdir_attachments(api, worker); return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh_async(&worker) - .await + .refresh(&worker) .map_err(ApiError::from) { - restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; - sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await?; - refresh_current_worker_session_locked(api, worker, &_session_guard).await?; + api.store.restore_worker_workdir_connection(&link)?; + sync_runtime_worker_workdir_attachments(api, worker)?; + refresh_current_worker_session_locked(api, worker).await?; return Err(error); } Ok(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -19534,36 +18578,41 @@ async fn detach_current_worker_workdir( }) } -async fn validated_current_worker_attachment_async( +fn validated_current_worker_attachment( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, - session_guard: Option<&Arc>>, ) -> ApiResult { - for imported in [false, true] { - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let stage_alias = target_workdir.to_string(); - let link = api + let find = || -> Result> { + Ok(api .store - .db_executor() - .run(move || -> Result<_> { - Ok(stage_api - .store - .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias)) - }) - .await?; - if let Some(link) = link { - return Ok(link); - } - if !imported { - // A Runtime may start the Worker before the outer spawn handler - // projects its reserved aliases. Import that exact set only once. - import_current_worker_workdir_observation(api, worker, session_guard).await?; + .list_worker_workdir_links(&api.config.workspace_id, worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == target_workdir)) + }; + if let Some(link) = find()? { + return Ok(link); + } + + // A Runtime can start the Worker immediately after reserving its local + // bindings, before the outer spawn handler projects them into the Backend. + // Import that exact alias set once; never substitute another attachment. + if !api + .store + .worker_workdir_link_history_exists(&api.config.workspace_id, worker)? + { + let observed_worker = api + .runtime + .worker(worker) + .map_err(|error| error.into_error())?; + if !observed_worker.workdir_attachments.is_empty() { + sync_worker_observation(api, &observed_worker)?; + if let Some(link) = find()? { + return Ok(link); + } } } + Err(Error::WorkdirAttachmentConflict(format!( "Worker {}:{} has no Workdir attachment alias `{target_workdir}`", worker.runtime_id, worker.worker_id @@ -19672,24 +18721,19 @@ async fn scoped_execute_current_worker_workdir_operation( context.stage = "workspace_scope"; validate_workspace_scope(&api, &path.workspace_id)?; context.stage = "worker_identity"; - let stage_api = api.clone(); - let worker = api.host_executors.host()?.run(move || { - current_worker_identity(&stage_api, &path.workspace_id, &headers) - }).await?; + let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; context.runtime_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.runtime_id)); context.worker_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.worker_id)); let target_workdir = request.target_workdir; let result = match request.operation { WorkdirSessionOperation::CommandStart(command) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = Arc::new(session_lock.lock_owned().await); + let _session_guard = session_lock.lock().await; context.stage = "attachment_validation"; - let link = validated_current_worker_attachment_async( - &api, &worker, &target_workdir, Some(&_session_guard), - ).await?; + let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; context.stage = "provider_dispatch"; let provider_handle = source .start_command(command) @@ -19721,7 +18765,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandStatus(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; session .command_status(provider_handle) .await @@ -19730,7 +18774,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandOutput(mut output) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context).await?; + current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context)?; output.handle = provider_handle; session .command_output(output) @@ -19740,7 +18784,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandCancel(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; session .cancel_command(provider_handle) .await @@ -19761,14 +18805,12 @@ async fn scoped_execute_current_worker_workdir_operation( | WorkdirSessionOperation::Glob(_) | WorkdirSessionOperation::Grep(_)) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = Arc::new(session_lock.lock_owned().await); + let _session_guard = session_lock.lock().await; context.stage = "attachment_validation"; - let link = validated_current_worker_attachment_async( - &api, &worker, &target_workdir, Some(&_session_guard), - ).await?; + let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; context.stage = "provider_dispatch"; execute_workdir_session_operation(&source, operation) .await @@ -19783,7 +18825,7 @@ async fn scoped_execute_current_worker_workdir_operation( }) } -async fn current_worker_command_session( +fn current_worker_command_session( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, @@ -19791,7 +18833,7 @@ async fn current_worker_command_session( context: &mut server_api::WorkdirOperationLogContext, ) -> std::result::Result<(WorkdirSessionHandle, CommandHandle), InternalWorkdirOperationError> { context.stage = "attachment_validation"; - let link = validated_current_worker_attachment_async(api, worker, target_workdir, None).await?; + let link = validated_current_worker_attachment(api, worker, target_workdir)?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "command_lookup"; let command = api @@ -19897,29 +18939,8 @@ async fn run_orchestrator_turn_end_hook(api: WorkspaceApi) { }; let mut worker_states = HashMap::new(); while let Some(update) = subscription.recv().await { - let stage_api = api.clone(); - let result = api.host_executors.host().map(|executor| async move { - executor - .run(move || -> Result<_> { - let keep_running = - observe_orchestrator_turn_end(&stage_api, &mut worker_states, update); - Ok((keep_running, worker_states)) - }) - .await - }); - match result { - Ok(operation) => match operation.await { - Ok((true, states)) => worker_states = states, - Ok((false, _)) => return, - Err(error) => { - tracing::warn!(%error, "Orchestrator hook stopped; durable inputs remain recoverable"); - return; - } - }, - Err(error) => { - tracing::warn!(%error, "Orchestrator hook executor unavailable"); - return; - } + if !observe_orchestrator_turn_end(&api, &mut worker_states, update) { + return; } } } @@ -19999,27 +19020,6 @@ fn maybe_dispatch_orchestrator_turn_end( } } -fn enqueue_orchestrator_assignment_attention( - api: &WorkspaceApi, -) -> Option> { - let api = api.clone(); - let (done, completion) = tokio::sync::oneshot::channel(); - match api.host_executors.host().and_then(|executor| { - executor.dispatch(move || { - if let Some(orchestrator) = find_workspace_orchestrator(&api) { - dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); - } - let _ = done.send(()); - }) - }) { - Ok(()) => Some(completion), - Err(error) => { - tracing::warn!(%error, "Assign notification was not admitted; input remains durable"); - None - } - } -} - fn dispatch_orchestrator_assignment_attention(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { with_orchestrator_attention_reservation(api, worker, true, |state, pending| { check_orchestrator_assignment_attention(api, worker, state, pending); @@ -20360,20 +19360,15 @@ async fn scoped_get_memory_document( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let document = api.authority.memory_document()?; - Ok(Json(MemoryDocumentResponse { - bytes: document.body_md.len(), - body_md: document.body_md, - created_at: document.created_at, - updated_at: document.updated_at, - record_source: document.record_source, - })) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let document = api.authority.memory_document()?; + Ok(Json(MemoryDocumentResponse { + bytes: document.body_md.len(), + body_md: document.body_md, + created_at: document.created_at, + updated_at: document.updated_at, + record_source: document.record_source, + })) } async fn scoped_reset_legacy_memory( @@ -20396,18 +19391,13 @@ async fn scoped_reset_legacy_memory( )) .into()); } - api.store - .db_executor() - .run(move || { - let counts = api.store.reset_legacy_memory(&path.workspace_id)?; - Ok(Json(server_api::LegacyMemoryResetResponse { - workspace_id: path.workspace_id, - documents_deleted: counts.documents, - staging_records_deleted: counts.staging_records, - staging_resolutions_deleted: counts.staging_resolutions, - })) - }) - .await + let counts = api.store.reset_legacy_memory(&path.workspace_id)?; + Ok(Json(server_api::LegacyMemoryResetResponse { + workspace_id: path.workspace_id, + documents_deleted: counts.documents, + staging_records_deleted: counts.staging_records, + staging_resolutions_deleted: counts.staging_resolutions, + })) } async fn scoped_list_memory_staging( @@ -20415,20 +19405,13 @@ async fn scoped_list_memory_staging( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(list_memory_staging_from_authority( - &api.authority, - query.limit.map(usize::try_from).transpose().map_err(|_| { - Error::InvalidInput( - "Memory staging limit exceeds this host's usize range".to_string(), - ) - })?, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(list_memory_staging_from_authority( + &api.authority, + query.limit.map(usize::try_from).transpose().map_err(|_| { + Error::InvalidInput("Memory staging limit exceeds this host's usize range".to_string()) + })?, + )?)) } async fn scoped_memory_backend_operation( @@ -20437,26 +19420,21 @@ async fn scoped_memory_backend_operation( _context: server_api::ServerRequestContext, Json(operation): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - // The legacy memory document edit is a read/replace/write across store - // calls. Keep that resource atomic without serializing unrelated requests. - let _memory_guard = api - .memory_operation_lock - .lock() - .expect("Memory operation lock poisoned"); - let result = execute_memory_backend_operation_with_authority(&api.authority, operation); - let response = match result { - Ok(result) => MemoryBackendHttpResponse::Ok { result }, - Err(error) => MemoryBackendHttpResponse::Error { - message: sanitize_backend_error(&error.to_string()), - }, - }; - Ok(Json(response)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + // The legacy memory document edit is a read/replace/write across store + // calls. Keep that resource atomic without serializing unrelated requests. + let _memory_guard = api + .memory_operation_lock + .lock() + .expect("Memory operation lock poisoned"); + let result = execute_memory_backend_operation_with_authority(&api.authority, operation); + let response = match result { + Ok(result) => MemoryBackendHttpResponse::Ok { result }, + Err(error) => MemoryBackendHttpResponse::Error { + message: sanitize_backend_error(&error.to_string()), + }, + }; + Ok(Json(response)) } const SUBJEKTIV_SINGLETON_PREFIX: &str = crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX; @@ -20464,66 +19442,6 @@ const SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX: &str = crate::hosts::SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX; const SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE: &str = "subjektiv-memory-consolidation"; -// Runtime callbacks may themselves need DB authority. Synchronous Host workflows -// therefore have a separate finite lane: never monopolize the sole DB worker while -// waiting for Runtime I/O. Existing transaction helpers still finish before I/O. -// Teardown waits must not occupy the ordinary Host lane: a Job may need a -// user stop/control operation on that lane before its Run can finish. Cleanup -// therefore has its own finite owner, still independent of the DB callback lane. -#[derive(Clone, Default)] -struct HostExecutors { - host: Arc>>, - cleanup: Arc>>, -} - -impl HostExecutors { - fn shared() -> Self { - static SHARED: std::sync::OnceLock = std::sync::OnceLock::new(); - SHARED.get_or_init(Self::default).clone() - } - - fn lane( - owner: &std::sync::OnceLock>, - ) -> Result { - owner - .get_or_init(|| { - crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { - workers: std::num::NonZeroUsize::new(2).unwrap(), - queued: std::num::NonZeroUsize::new(16).unwrap(), - }) - .ok() - }) - .clone() - .ok_or(Error::DbExecutorUnavailable) - } - - fn host(&self) -> Result { - Self::lane(&self.host) - } - fn cleanup(&self) -> Result { - Self::lane(&self.cleanup) - } -} - -// Some existing lifecycle workflows interleave owned async resource locks with -// synchronous DB/Runtime APIs. Poll the complete workflow on the finite Host -// lane using the original runtime (never create a replacement runtime for its -// cached services). Cancellation releases neither locks nor accepted ownership. -// No SQL transaction contains an await/Runtime call, and this lane is separate -// from the DB worker needed by Runtime callbacks. Do not nest this helper. -async fn run_host_workflow( - executor: crate::db_executor::DbExecutor, - future: impl std::future::Future> + Send + 'static, -) -> std::result::Result -where - T: Send + 'static, - E: From + Send + 'static, -{ - let runtime = - tokio::runtime::Handle::try_current().map_err(|_| E::from(Error::DbExecutorUnavailable))?; - executor.run(move || runtime.block_on(future)).await -} - fn open_subjektiv_store(api: &WorkspaceApi) -> ApiResult { crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) .map_err(|error| ApiError::from(Error::Store(error.to_string()))) @@ -20741,23 +19659,18 @@ async fn scoped_create_subjektiv_subject( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = crate::subjektiv::SubjectRole::new(request.role) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .create_subject_with_behavior(role, request.behavior_md) - .map_err(subjektiv_store_error)?; - Ok(( - StatusCode::CREATED, - Json(subjektiv_subject_response(&api, subject)?), - )) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let role = crate::subjektiv::SubjectRole::new(request.role) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .create_subject_with_behavior(role, request.behavior_md) + .map_err(subjektiv_store_error)?; + Ok(( + StatusCode::CREATED, + Json(subjektiv_subject_response(&api, subject)?), + )) } #[derive(Debug, Serialize, Deserialize)] @@ -20773,86 +19686,76 @@ async fn scoped_list_subjektiv_subjects( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let limit = query.limit.unwrap_or(20); - if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { - return Err(Error::InvalidInput(format!( - "subject list limit must be within 1..={}", - server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT - )) - .into()); - } - let cursor = query - .cursor - .as_deref() - .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) - .transpose()?; - if cursor + validate_workspace_scope(&api, &path.workspace_id)?; + let limit = query.limit.unwrap_or(20); + if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { + return Err(Error::InvalidInput(format!( + "subject list limit must be within 1..={}", + server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT + )) + .into()); + } + let cursor = query + .cursor + .as_deref() + .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) + .transpose()?; + if cursor + .as_ref() + .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) + { + return Err(Error::InvalidInput( + "subjektiv subject cursor does not match Workspace".to_string(), + ) + .into()); + } + let page = open_subjektiv_store(&api)? + .list_subjects_after( + limit, + cursor .as_ref() - .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) - { - return Err(Error::InvalidInput( - "subjektiv subject cursor does not match Workspace".to_string(), + .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), + ) + .map_err(subjektiv_store_error)?; + let next_cursor = if page.has_more { + page.items + .last() + .map(|subject| { + encode_subjektiv_cursor( + "subjects", + &SubjektivSubjectCursor { + workspace_id: path.workspace_id.clone(), + created_at: subject.created_at.clone(), + subject_id: subject.id.clone(), + }, ) - .into()); - } - let page = open_subjektiv_store(&api)? - .list_subjects_after( - limit, - cursor - .as_ref() - .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), - ) - .map_err(subjektiv_store_error)?; - let next_cursor = if page.has_more { - page.items - .last() - .map(|subject| { - encode_subjektiv_cursor( - "subjects", - &SubjektivSubjectCursor { - workspace_id: path.workspace_id.clone(), - created_at: subject.created_at.clone(), - subject_id: subject.id.clone(), - }, - ) - }) - .transpose()? - } else { - None - }; - Ok(Json(server_api::SubjektivSubjectListResponse { - limit, - items: page - .items - .into_iter() - .map(|subject| subjektiv_subject_response(&api, subject)) - .collect::>>()?, - next_cursor, - has_more: page.has_more, - })) - }) - .await + }) + .transpose()? + } else { + None + }; + Ok(Json(server_api::SubjektivSubjectListResponse { + limit, + items: page + .items + .into_iter() + .map(|subject| subjektiv_subject_response(&api, subject)) + .collect::>>()?, + next_cursor, + has_more: page.has_more, + })) } async fn scoped_get_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .subject(&path.subject_id) - .map_err(|error| Error::Store(error.to_string()))? - .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .subject(&path.subject_id) + .map_err(|error| Error::Store(error.to_string()))? + .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) } async fn scoped_update_subjektiv_subject_behavior( @@ -20860,22 +19763,17 @@ async fn scoped_update_subjektiv_subject_behavior( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .update_subject_behavior( - &path.subject_id, - &request.expected_behavior_md, - request.behavior_md, - ) - .map_err(subjektiv_store_error)?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .update_subject_behavior( + &path.subject_id, + &request.expected_behavior_md, + request.behavior_md, + ) + .map_err(subjektiv_store_error)?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) } fn require_subjektiv_subject( @@ -20896,53 +19794,47 @@ async fn scoped_get_subjektiv_resident_surface( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - let resident = store - .resident_surface(&path.subject_id) - .map_err(subjektiv_store_error)?; - let availability = match resident.availability { - crate::subjektiv::SurfaceAvailability::Ungenerated => { - server_api::SubjektivResidentSurfaceAvailability::Ungenerated - } - crate::subjektiv::SurfaceAvailability::Stale => { - server_api::SubjektivResidentSurfaceAvailability::Stale - } - crate::subjektiv::SurfaceAvailability::Failed => { - server_api::SubjektivResidentSurfaceAvailability::Failed - } - crate::subjektiv::SurfaceAvailability::Ready => { - server_api::SubjektivResidentSurfaceAvailability::Ready - } - }; - let snapshot = - resident - .snapshot - .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { - snapshot_id: snapshot.id, - body_md: snapshot.body_md, - memory_refs: snapshot - .memory_refs - .into_iter() - .map(|reference| server_api::SubjektivMemoryChangeRef { - memory_id: reference.memory_id, - change_id: reference.change_id, - }) - .collect(), - built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, - created_at: snapshot.created_at, - }); - Ok(Json(server_api::SubjektivResidentSurfaceResponse { - subject_id: path.subject_id, - availability, - snapshot, - })) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + let resident = store + .resident_surface(&path.subject_id) + .map_err(subjektiv_store_error)?; + let availability = match resident.availability { + crate::subjektiv::SurfaceAvailability::Ungenerated => { + server_api::SubjektivResidentSurfaceAvailability::Ungenerated + } + crate::subjektiv::SurfaceAvailability::Stale => { + server_api::SubjektivResidentSurfaceAvailability::Stale + } + crate::subjektiv::SurfaceAvailability::Failed => { + server_api::SubjektivResidentSurfaceAvailability::Failed + } + crate::subjektiv::SurfaceAvailability::Ready => { + server_api::SubjektivResidentSurfaceAvailability::Ready + } + }; + let snapshot = resident + .snapshot + .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { + snapshot_id: snapshot.id, + body_md: snapshot.body_md, + memory_refs: snapshot + .memory_refs + .into_iter() + .map(|reference| server_api::SubjektivMemoryChangeRef { + memory_id: reference.memory_id, + change_id: reference.change_id, + }) + .collect(), + built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, + created_at: snapshot.created_at, + }); + Ok(Json(server_api::SubjektivResidentSurfaceResponse { + subject_id: path.subject_id, + availability, + snapshot, + })) } async fn scoped_list_subjektiv_memories( @@ -20950,19 +19842,14 @@ async fn scoped_list_subjektiv_memories( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_query( - &store, - &path.subject_id, - query, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_query( + &store, + &path.subject_id, + query, + )?)) } async fn scoped_get_subjektiv_memory( @@ -20970,26 +19857,21 @@ async fn scoped_get_subjektiv_memory( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_read( - &store, - &path.subject_id, - server_api::SubjektivMemoryReadRequest { - memory_id: path.memory_id, - change_id: query.change_id, - offset: query.offset, - byte_offset: query.byte_offset, - limit: query.limit, - evidence_cursor: query.evidence_cursor, - }, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_read( + &store, + &path.subject_id, + server_api::SubjektivMemoryReadRequest { + memory_id: path.memory_id, + change_id: query.change_id, + offset: query.offset, + byte_offset: query.byte_offset, + limit: query.limit, + evidence_cursor: query.evidence_cursor, + }, + )?)) } async fn scoped_list_subjektiv_memory_changes( @@ -20997,41 +19879,31 @@ async fn scoped_list_subjektiv_memory_changes( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_list_changes( - &store, - &path.subject_id, - server_api::SubjektivMemoryListChangesRequest { - memory_id: path.memory_id, - limit: query.limit, - cursor: query.cursor, - }, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_list_changes( + &store, + &path.subject_id, + server_api::SubjektivMemoryListChangesRequest { + memory_id: path.memory_id, + limit: query.limit, + cursor: query.cursor, + }, + )?)) } async fn scoped_retire_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .retire_subject(&path.subject_id) - .map_err(subjektiv_store_error)?; - // Retirement deliberately does not stop or remove the current Worker. The - // Worker lease and subject lifecycle are independent authorities. - Ok(Json(subjektiv_subject_response(&api, subject)?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .retire_subject(&path.subject_id) + .map_err(subjektiv_store_error)?; + // Retirement deliberately does not stop or remove the current Worker. The + // Worker lease and subject lifecycle are independent authorities. + Ok(Json(subjektiv_subject_response(&api, subject)?)) } async fn scoped_stage_subjektiv_candidate( @@ -21040,63 +19912,58 @@ async fn scoped_stage_subjektiv_candidate( context: server_api::ServerRequestContext, Json(mut request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )?; - let subject_id = attribution.subject_id.clone(); - let session_id = attribution.session_id.as_str(); - for source_ref in &mut request.operation.source_refs { - if source_ref - .session_id - .as_deref() - .is_some_and(|value| value != session_id) - { - return Err(Error::InvalidInput( - "candidate source_ref belongs to a different Session".to_string(), - ) - .into()); - } - source_ref.session_id = Some(session_id.to_string()); - } - if matches!( - request.operation.candidate.kind, - memory::extract::CandidateKind::Preference - ) && request.operation.evidence.iter().any(|evidence| { - !matches!( - evidence.origin.as_ref().map(|origin| &origin.kind), - Some(memory::schema::EvidenceOriginKind::HumanInput) - ) - }) { - return Err(Error::InvalidInput( - "preference candidates require exclusively HumanInput evidence".to_string(), - ) - .into()); - } - let candidate_id = Uuid::now_v7().to_string(); - let record = memory::extract::StagingRecord::from_candidate( - candidate_id, - request.operation.extract_run_id, - request.operation.source, - request.operation.candidate, - request.operation.evidence, - request.operation.source_refs, - ); - let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); - let (staged, _) = open_subjektiv_store(&api)? - .stage_candidate_with_attribution(record, attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - Ok(Json(server_api::SubjektivStageCandidateResponse { - staging_id: staged.id, - })) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )?; + let subject_id = attribution.subject_id.clone(); + let session_id = attribution.session_id.as_str(); + for source_ref in &mut request.operation.source_refs { + if source_ref + .session_id + .as_deref() + .is_some_and(|value| value != session_id) + { + return Err(Error::InvalidInput( + "candidate source_ref belongs to a different Session".to_string(), + ) + .into()); + } + source_ref.session_id = Some(session_id.to_string()); + } + if matches!( + request.operation.candidate.kind, + memory::extract::CandidateKind::Preference + ) && request.operation.evidence.iter().any(|evidence| { + !matches!( + evidence.origin.as_ref().map(|origin| &origin.kind), + Some(memory::schema::EvidenceOriginKind::HumanInput) + ) + }) { + return Err(Error::InvalidInput( + "preference candidates require exclusively HumanInput evidence".to_string(), + ) + .into()); + } + let candidate_id = Uuid::now_v7().to_string(); + let record = memory::extract::StagingRecord::from_candidate( + candidate_id, + request.operation.extract_run_id, + request.operation.source, + request.operation.candidate, + request.operation.evidence, + request.operation.source_refs, + ); + let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); + let (staged, _) = open_subjektiv_store(&api)? + .stage_candidate_with_attribution(record, attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + Ok(Json(server_api::SubjektivStageCandidateResponse { + staging_id: staged.id, + })) } async fn scoped_record_subjektiv_session( @@ -21105,34 +19972,25 @@ async fn scoped_record_subjektiv_session( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = if request.create_if_missing { - let attribution = subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )?; - open_subjektiv_store(&api)? - .record_session_attribution(attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))? - } else { - require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )? - }; - Ok(Json(server_api::SubjektivRecordSessionResponse { - subject_id: attribution.subject_id, - session_id: attribution.session_id, - })) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = if request.create_if_missing { + let attribution = + subjektiv_session_attribution(&api, &path.workspace_id, &context, &request.session_id)?; + open_subjektiv_store(&api)? + .record_session_attribution(attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))? + } else { + require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )? + }; + Ok(Json(server_api::SubjektivRecordSessionResponse { + subject_id: attribution.subject_id, + session_id: attribution.session_id, + })) } const SUBJEKTIV_SESSION_DEFAULT_LIMIT: usize = 20; @@ -21226,61 +20084,50 @@ async fn scoped_subjektiv_session_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, controller, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - require_subjektiv_worker_authority( - authority, - SubjektivWorkerAuthority::Subject, - "Session history read", - )?; - let store = match open_subjektiv_store(&api) { - Ok(store) => store, - Err(error) => { - return Ok(Json(subjektiv_session_error( - server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, - error.error.to_string(), - ))); - } - }; - let response = match request.operation { - server_api::SubjektivSessionBackendOperation::List(input) => { - subjektiv_session_list( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ) - } - server_api::SubjektivSessionBackendOperation::Search(input) => { - subjektiv_session_search( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ) - } - server_api::SubjektivSessionBackendOperation::Read(input) => { - subjektiv_session_read( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ) - } - }; - Ok(Json(response)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, controller, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + require_subjektiv_worker_authority( + authority, + SubjektivWorkerAuthority::Subject, + "Session history read", + )?; + let store = match open_subjektiv_store(&api) { + Ok(store) => store, + Err(error) => { + return Ok(Json(subjektiv_session_error( + server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, + error.error.to_string(), + ))); + } + }; + let response = match request.operation { + server_api::SubjektivSessionBackendOperation::List(input) => subjektiv_session_list( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ), + server_api::SubjektivSessionBackendOperation::Search(input) => subjektiv_session_search( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ), + server_api::SubjektivSessionBackendOperation::Read(input) => subjektiv_session_read( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ), + }; + Ok(Json(response)) } fn subjektiv_session_list( @@ -22471,121 +21318,115 @@ async fn scoped_subjektiv_memory_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, source_worker, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - let store = open_subjektiv_store(&api)?; - use server_api::SubjektivMemoryBackendOperation as Op; - // Authentication, live Worker/Job authority and committed Session provenance - // remain Backend-owned. Only the domain operation/projection is shared. - let (expected, label) = match &request.operation { - Op::ResidentSummary(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "resident surface read", - ), - Op::ResidentContext(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "resident subject context read", - ), - Op::ValidateProposal(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "Memory proposal validation", - ), - Op::ReceiptStatus(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "explicit candidate receipt lookup", - ), - Op::StageExplicit(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "explicit candidate staging", - ), - Op::ListCandidates(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate listing", - ), - Op::ReadCandidate(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate read", - ), - Op::DecideCandidate(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate decision", - ), - Op::PrepareSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface generation preparation", - ), - Op::PublishSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface publication", - ), - Op::FailSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface generation failure", - ), - Op::Query(_) | Op::Read(_) | Op::ListChanges(_) => (None, "recall"), + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, source_worker, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + let store = open_subjektiv_store(&api)?; + use server_api::SubjektivMemoryBackendOperation as Op; + // Authentication, live Worker/Job authority and committed Session provenance + // remain Backend-owned. Only the domain operation/projection is shared. + let (expected, label) = match &request.operation { + Op::ResidentSummary(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "resident surface read", + ), + Op::ResidentContext(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "resident subject context read", + ), + Op::ValidateProposal(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "Memory proposal validation", + ), + Op::ReceiptStatus(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "explicit candidate receipt lookup", + ), + Op::StageExplicit(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "explicit candidate staging", + ), + Op::ListCandidates(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate listing", + ), + Op::ReadCandidate(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate read", + ), + Op::DecideCandidate(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate decision", + ), + Op::PrepareSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface generation preparation", + ), + Op::PublishSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface publication", + ), + Op::FailSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface generation failure", + ), + Op::Query(_) | Op::Read(_) | Op::ListChanges(_) => (None, "recall"), + }; + if let Some(expected) = expected { + require_subjektiv_worker_authority(authority, expected, label)?; + } + let host_context = match authority { + SubjektivWorkerAuthority::Subject => { + let attribution = if let Op::StageExplicit(input) = &request.operation { + Some(require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &input.session_id, + )?) + } else { + None }; - if let Some(expected) = expected { - require_subjektiv_worker_authority(authority, expected, label)?; + HostOperationContext::validated_body(&store, &subject_id, attribution)? + } + SubjektivWorkerAuthority::Consolidation => { + match &request.operation { + Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + _ => {} } - let host_context = match authority { - SubjektivWorkerAuthority::Subject => { - let attribution = if let Op::StageExplicit(input) = &request.operation { - Some(require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &input.session_id, - )?) - } else { - None - }; - HostOperationContext::validated_body(&store, &subject_id, attribution)? - } - SubjektivWorkerAuthority::Consolidation => { - match &request.operation { - Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - _ => {} - } - let grant = - subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; - let job = api - .store - .worker_registry_projection(&path.workspace_id, &source_worker)? - .and_then(|projection| projection.job) - .map(|binding| JobAttemptBinding { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - }); - HostOperationContext::validated_consolidation( - &store, - &subject_id, - grant.map(|g| g.candidate_ids), - job, - )? - } - }; - Ok(Json(subjektiv::api::execute( + let grant = subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; + let job = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|projection| projection.job) + .map(|binding| JobAttemptBinding { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + }); + HostOperationContext::validated_consolidation( &store, - &host_context, - request.operation, - )?)) - }) - .await + &subject_id, + grant.map(|g| g.candidate_ids), + job, + )? + } + }; + Ok(Json(subjektiv::api::execute( + &store, + &host_context, + request.operation, + )?)) } fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error { @@ -22689,17 +21530,10 @@ async fn start_subjektiv_staging_consolidation( subject_id: &str, operation: MemoryConsolidateStagingOperation, ) -> ApiResult { - let job_api = api.clone(); - let subject = subject_id.to_owned(); - let (candidate_count, total_bytes) = api - .store - .db_executor() - .run(move || { - open_subjektiv_store(&job_api)? - .pending_staging_backlog(&subject) - .map_err(|error| ApiError::from(subjektiv_store_error(error))) - }) - .await?; + let store = open_subjektiv_store(&api)?; + let (candidate_count, total_bytes) = store + .pending_staging_backlog(subject_id) + .map_err(subjektiv_store_error)?; if !subjektiv_job::retire_legacy_worker(&api, subject_id).await? { return Ok(MemoryConsolidationOutput { status: "skipped_legacy_inflight".into(), @@ -22707,18 +21541,10 @@ async fn start_subjektiv_staging_consolidation( candidate_count, total_bytes, }); } - let job_api = api.clone(); - let subject = subject_id.to_owned(); - let surface_availability = api - .store - .db_executor() - .run(move || { - open_subjektiv_store(&job_api)? - .resident_surface(&subject) - .map(|surface| surface.availability) - .map_err(|error| ApiError::from(subjektiv_store_error(error))) - }) - .await?; + let surface_availability = store + .resident_surface(subject_id) + .map_err(subjektiv_store_error)? + .availability; if !subject_consolidation_required( candidate_count, total_bytes, @@ -22745,11 +21571,7 @@ async fn start_subjektiv_staging_consolidation( }); } - let subject = subject_id.to_owned(); - api.host_executors - .host()? - .run(move || subjektiv_job::dispatch(&api, &subject, candidate_count, total_bytes)) - .await + subjektiv_job::dispatch(&api, subject_id, candidate_count, total_bytes) } async fn scoped_memory_consolidation( @@ -22757,13 +21579,8 @@ async fn scoped_memory_consolidation( AxumPath(path): AxumPath, Json(operation): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(start_memory_staging_consolidation(api, operation)?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(start_memory_staging_consolidation(api, operation)?)) } fn start_memory_staging_consolidation( @@ -22963,92 +21780,72 @@ async fn scoped_list_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::catalog(&state).map(Json).map_err(skill_api_error) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::catalog(&state).map(Json).map_err(skill_api_error) } async fn scoped_lint_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::lint(&state).map(Json).map_err(skill_api_error) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::lint(&state).map(Json).map_err(skill_api_error) } async fn scoped_get_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::detail(&state, &path.name) - .map(Json) - .map_err(skill_api_error) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::detail(&state, &path.name) + .map(Json) + .map_err(skill_api_error) } async fn scoped_activate_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::activation(&state, &path.name) - .map(Json) - .map_err(skill_api_error) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::activation(&state, &path.name) + .map(Json) + .map_err(skill_api_error) } fn skill_api_error(error: skills::SkillError) -> ApiError { @@ -23098,13 +21895,8 @@ async fn scoped_query_objectives( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_objectives(query)?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_objectives(query)?)) } async fn scoped_show_objective( @@ -23112,15 +21904,10 @@ async fn scoped_show_objective( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority.show_objective(&path.objective_id, query)?, - )) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority.show_objective(&path.objective_id, query)?, + )) } async fn scoped_create_objective( @@ -23128,20 +21915,15 @@ async fn scoped_create_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.create_objective( - ObjectiveCreateInput { - title: request.title, - body_md: request.body_md, - state: request.state, - linked_tickets: request.linked_tickets, - }, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.create_objective( + ObjectiveCreateInput { + title: request.title, + body_md: request.body_md, + state: request.state, + linked_tickets: request.linked_tickets, + }, + )?)) } async fn scoped_edit_objective( @@ -23149,21 +21931,16 @@ async fn scoped_edit_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.edit_objective( - &path.objective_id, - ObjectiveEditInput { - title: request.title, - old_string: request.old_string, - new_string: request.new_string, - replace_all: request.replace_all, - }, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.edit_objective( + &path.objective_id, + ObjectiveEditInput { + title: request.title, + old_string: request.old_string, + new_string: request.new_string, + replace_all: request.replace_all, + }, + )?)) } async fn scoped_set_objective_state( @@ -23171,16 +21948,11 @@ async fn scoped_set_objective_state( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority - .set_objective_state(&path.objective_id, &request.state)?, - )) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority + .set_objective_state(&path.objective_id, &request.state)?, + )) } async fn scoped_link_objective_ticket( @@ -23188,32 +21960,22 @@ async fn scoped_link_objective_ticket( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.link_objective_ticket( - &path.objective_id, - &request.ticket_id, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.link_objective_ticket( + &path.objective_id, + &request.ticket_id, + )?)) } async fn scoped_unlink_objective_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.unlink_objective_ticket( - &path.objective_id, - &path.ticket_id, - )?)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.unlink_objective_ticket( + &path.objective_id, + &path.ticket_id, + )?)) } async fn scoped_list_repositories( @@ -23721,9 +22483,6 @@ async fn worker_remove_contract( .map_err(|error| { let status = match error { crate::worker_source::WorkerMutationSourceProofError::Replay => StatusCode::CONFLICT, - crate::worker_source::WorkerMutationSourceProofError::Unavailable => { - StatusCode::SERVICE_UNAVAILABLE - } crate::worker_source::WorkerMutationSourceProofError::Authority(_) => { StatusCode::INTERNAL_SERVER_ERROR } @@ -23738,8 +22497,7 @@ async fn worker_remove_contract( ) })?; let worker = - WorkspaceWorker::resolve_async(api, &request.target_runtime_id, &request.target_worker_id) - .await + WorkspaceWorker::resolve(api, &request.target_runtime_id, &request.target_worker_id) .map_err(|error| match error { Error::UnknownWorker { .. } => server_api::WorkerRemoveApiError::new( StatusCode::NOT_FOUND.as_u16(), @@ -23795,28 +22553,22 @@ async fn scoped_get_workspace_worker( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker_ref = api - .store - .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? - .ok_or_else(|| Error::UnknownWorker { - worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), - })?; - let workers = workers_response(api.clone())?; - workers - .items - .into_iter() - .find(|worker| { - worker.runtime_id == worker_ref.runtime_id - && worker.worker_id == worker_ref.worker_id - }) - .map(Json) - .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) + validate_workspace_scope(&api, &path.workspace_id)?; + let worker_ref = api + .store + .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? + .ok_or_else(|| Error::UnknownWorker { + worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), + })?; + let workers = workers_response(api.clone())?; + workers + .items + .into_iter() + .find(|worker| { + worker.runtime_id == worker_ref.runtime_id && worker.worker_id == worker_ref.worker_id }) - .await + .map(Json) + .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) } async fn discover_workspace_workers( @@ -23939,13 +22691,8 @@ async fn scoped_workspace_orchestrator_status( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(workspace_orchestrator_response(&api, "observed"))) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(workspace_orchestrator_response(&api, "observed"))) } async fn list_known_workers( @@ -23953,43 +22700,36 @@ async fn list_known_workers( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let limit = api.config.max_records.clamp(1, 500); - let grants = api.store.list_active_worker_control_grants( - &path.workspace_id, - &controller, - limit + 1, - )?; - let truncated = grants.len() > limit; - let mut items = Vec::with_capacity(grants.len().min(limit)); - for grant in grants.into_iter().take(limit) { - let summary = api - .runtime - .worker(&grant.subject) - .map_err(|error| error.into_error())?; - items.push(server_api::WorkerControlRecord { - subject: server_api::WorkerControlSubject { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - relation: grant.relation, - origin: grant.origin, - permissions: grant.permissions, - summary: worker_launch_worker_summary(summary), - }); - } - Ok(Json(server_api::WorkerControlListResponse { - workspace_id: path.workspace_id, - items, - truncated, - })) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let limit = api.config.max_records.clamp(1, 500); + let grants = + api.store + .list_active_worker_control_grants(&path.workspace_id, &controller, limit + 1)?; + let truncated = grants.len() > limit; + let mut items = Vec::with_capacity(grants.len().min(limit)); + for grant in grants.into_iter().take(limit) { + let summary = api + .runtime + .worker(&grant.subject) + .map_err(|error| error.into_error())?; + items.push(server_api::WorkerControlRecord { + subject: server_api::WorkerControlSubject { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + relation: grant.relation, + origin: grant.origin, + permissions: grant.permissions, + summary: worker_launch_worker_summary(summary), + }); + } + Ok(Json(server_api::WorkerControlListResponse { + workspace_id: path.workspace_id, + items, + truncated, + })) } fn scoped_worker_control_operation_id(controller: &RuntimeWorkerRef, operation_id: &str) -> String { @@ -24010,27 +22750,6 @@ async fn spawn_known_worker( } async fn spawn_known_worker_authenticated( - State(api): State, - AxumPath(path): AxumPath, - headers: HeaderMap, - Json(request): Json, - context: Option<&server_api::ServerRequestContext>, -) -> ApiResult> { - let context = context.cloned(); - run_host_workflow(api.host_executors.host()?, async move { - spawn_known_worker_authenticated_on_host( - State(api), - AxumPath(path), - headers, - Json(request), - context.as_ref(), - ) - .await - }) - .await -} - -async fn spawn_known_worker_authenticated_on_host( State(api): State, AxumPath(path): AxumPath, headers: HeaderMap, @@ -24073,7 +22792,7 @@ async fn spawn_known_worker_authenticated_on_host( operation_id: scoped_worker_control_operation_id(&controller, &operation_id), input_fingerprint, }); - let response = create_workspace_worker_authenticated_on_host( + let response = create_workspace_worker_authenticated( api.clone(), headers, request, @@ -24161,9 +22880,7 @@ async fn send_known_worker_input( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = - WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) - .await?; + let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; let request = worker_input_request_from_api(request)?; let idle_only = request.kind == WorkerInputKind::User; let result = worker.input(&context, request).await?; @@ -24199,9 +22916,7 @@ async fn cancel_known_worker( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = - WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) - .await?; + let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; Ok(Json(worker_lifecycle_result_to_api( worker .cancel(&context, worker_lifecycle_request_from_api(request)) @@ -24220,9 +22935,7 @@ async fn stop_known_worker( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = - WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) - .await?; + let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; Ok(Json(worker_lifecycle_result_to_api( worker .stop(&context, worker_lifecycle_request_from_api(request)) @@ -24256,46 +22969,41 @@ async fn scoped_list_worker_observation_sessions( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let sessions = api - .store - .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? - .into_iter() - .filter(|grant| { - grant - .permissions - .iter() - .any(|permission| permission == "observe") - }) - .filter_map(|grant| { - let worker = api.runtime.worker(&grant.subject).ok()?; - if matches!( - worker.state.as_str(), - "stopped" | "failed" | "rejected" | "disconnected" - ) { - return None; - } - Some(server_api::WorkerObservationSubject { - subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - display_name: worker.display_name, - relation: grant.relation, - status: worker.state, - }) - }) - .collect::>(); - Ok(Json(server_api::WorkerObservationSessionsResponse { - sessions, - })) + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let sessions = api + .store + .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? + .into_iter() + .filter(|grant| { + grant + .permissions + .iter() + .any(|permission| permission == "observe") }) - .await + .filter_map(|grant| { + let worker = api.runtime.worker(&grant.subject).ok()?; + if matches!( + worker.state.as_str(), + "stopped" | "failed" | "rejected" | "disconnected" + ) { + return None; + } + Some(server_api::WorkerObservationSubject { + subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + display_name: worker.display_name, + relation: grant.relation, + status: worker.state, + }) + }) + .collect::>(); + Ok(Json(server_api::WorkerObservationSessionsResponse { + sessions, + })) } async fn scoped_capture_worker_observation_session( @@ -24383,7 +23091,6 @@ async fn scoped_start_workspace_orchestrator( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.host_executors.host()?.run(move || { #[cfg(test)] tests::seed_test_delivery_binding(&api); validate_workspace_scope(&api, &path.workspace_id)?; @@ -24510,8 +23217,6 @@ async fn scoped_start_workspace_orchestrator( drop(_guard); dispatch_orchestrator_assignment_attention(&api, &worker.worker); Ok(Json(workspace_orchestrator_response(&api, disposition))) - - }).await } fn worker_launch_worker_summary(worker: InternalWorkerSummary) -> WorkerLaunchWorkerSummary { @@ -24644,18 +23349,13 @@ async fn scoped_list_runtime_working_directories( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - api.host_executors - .host()? - .run(move || { - let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) - }) - .await + let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) } async fn scoped_create_runtime_working_directory( @@ -24679,12 +23379,7 @@ async fn scoped_runtime_working_directory_detail( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - api.host_executors - .host()? - .run(move || { - working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) - }) - .await + working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) } fn workdir_removal_source_actor( @@ -24770,25 +23465,19 @@ async fn cleanup_runtime_working_directory( ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - api.host_executors - .host()? - .run(move || { - let registered_runtime = - registered_workdir_runtime_id(&api, &path.working_directory_id)?; - if registered_runtime != path.runtime_id { - return Err(ApiError::from(Error::WorkspacePermissionDenied( - "Workdir does not belong to the requested Runtime".to_string(), - ))); - } - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) - }) - .await + let registered_runtime = registered_workdir_runtime_id(&api, &path.working_directory_id)?; + if registered_runtime != path.runtime_id { + return Err(ApiError::from(Error::WorkspacePermissionDenied( + "Workdir does not belong to the requested Runtime".to_string(), + ))); + } + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) } fn external_workdir_expires_at( @@ -24840,18 +23529,9 @@ fn schedule_external_workdir_expiry( } Err(error) => { tracing::error!(grant_id = %task_grant_id, error = ?error, "Backend External Workdir expiry cleanup requires retry"); - let stage_api = task_api.clone(); - let stage_grant_id = task_grant_id.clone(); let retryable = task_api .store - .db_executor() - .run(move || { - stage_api.store.get_external_workdir_grant( - &stage_api.config.workspace_id, - &stage_grant_id, - ) - }) - .await + .get_external_workdir_grant(&task_api.config.workspace_id, &task_grant_id) .map_or(true, |grant| grant.is_some()); if !retryable { break; @@ -24916,18 +23596,6 @@ async fn stop_external_workdir_provider( async fn fence_external_workdir_grant_expired( api: &WorkspaceApi, grant_id: &str, -) -> ApiResult { - let stage_api = api.clone(); - let grant_id = grant_id.to_string(); - api.store - .db_executor() - .run(move || fence_external_workdir_grant_expired_stage(&stage_api, &grant_id)) - .await -} - -fn fence_external_workdir_grant_expired_stage( - api: &WorkspaceApi, - grant_id: &str, ) -> ApiResult { loop { let mut grant = api @@ -24997,8 +23665,6 @@ async fn scoped_create_external_workdir_grant( Extension(actor): Extension, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - api.store.db_executor().run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; let display_name = request.display_name.trim(); if display_name.is_empty() @@ -25068,8 +23734,6 @@ async fn scoped_create_external_workdir_grant( StatusCode::CREATED, Json(external_workdir_grant_response(&grant)?), )) - - }).await } fn external_workdir_grant_response( @@ -25094,21 +23758,16 @@ async fn scoped_external_workdir_provider_ws( AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ws: WebSocketUpgrade, ) -> ApiResult { - let stage_api = api.clone(); - let stage_workspace_id = workspace_id.clone(); - let stage_grant_id = grant_id.clone(); - let mut grant = api.store.db_executor().run(move || -> ApiResult<_> { - validate_workspace_scope(&stage_api, &stage_workspace_id)?; - let grant = stage_api.store - .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir provider connection must use the grant creator's authenticated client".to_string(), - ).into()); - } - Ok(grant) - }).await?; + validate_workspace_scope(&api, &workspace_id)?; + let mut grant = api + .store + .get_external_workdir_grant(&workspace_id, &grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir provider connection must use the grant creator's authenticated client".to_string(), + ).into()); + } let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(&api, &grant_id).await?; @@ -25255,102 +23914,81 @@ async fn serve_external_workdir_provider( .await; return; } - let stage_api = api.clone(); - let stage_grant = initial_grant.clone(); - let registration_generation = registration.generation; - let registration_capabilities = registration.capabilities; - let activation = api + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !api .store - .db_executor() - .run(move || -> Result<_> { - let api = stage_api; - let initial_grant = stage_grant; - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !api - .store - .activate_external_workdir_grant( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration_generation, - &now, - ) - .unwrap_or(false) - { - return Ok(None); - } - if let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_connected", - &initial_grant.grant_id, - None, - "online", - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, registration_generation - ), - &now, - ) { - let _ = api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration_generation, - "offline", - &now, - ); - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = registration_generation, - %error, - "failed to audit External Workdir provider activation" - ); - return Ok(None); - } - let expires_at = match external_workdir_expires_at(&initial_grant) { - Ok(value) => value, - Err(_) => return Ok(None), - }; - let (sender, commands) = tokio::sync::mpsc::channel(32); - let connection = Arc::new(ExternalProviderConnection { - grant_id: initial_grant.grant_id.clone(), - workdir_id: initial_grant.workdir_id.clone(), - provider_instance_id: initial_grant.provider_instance_id.clone(), - generation: registration_generation, - expires_at, - capabilities: registration_capabilities, - admission: Arc::new(tokio::sync::Semaphore::new(16)), - shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), - sender, - }); - // Publish with the accepted activation stage, never in its cancellable waiter. - if api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connected(connection.clone()) - .is_some() - { - return Ok(None); - } - Ok(Some((connection, commands))) - }) - .await; - let (connection, mut commands) = match activation { - Ok(Some(activated)) => activated, - Ok(None) => { - let _ = socket.close().await; - return; - } - Err(error) => { - tracing::error!(grant_id = %initial_grant.grant_id, %error, - "External Workdir provider activation stage failed"); + .activate_external_workdir_grant( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration.generation, + &now, + ) + .unwrap_or(false) + { + let _ = socket.close().await; + return; + } + if let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_connected", + &initial_grant.grant_id, + None, + "online", + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, registration.generation + ), + &now, + ) { + let _ = api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration.generation, + "offline", + &now, + ); + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = registration.generation, + %error, + "failed to audit External Workdir provider activation" + ); + let _ = socket.close().await; + return; + } + let expires_at = match external_workdir_expires_at(&initial_grant) { + Ok(value) => value, + Err(_) => { let _ = socket.close().await; return; } }; + let (sender, mut commands) = tokio::sync::mpsc::channel(32); + let connection = Arc::new(ExternalProviderConnection { + grant_id: initial_grant.grant_id.clone(), + workdir_id: initial_grant.workdir_id.clone(), + provider_instance_id: initial_grant.provider_instance_id.clone(), + generation: registration.generation, + expires_at, + capabilities: registration.capabilities, + admission: Arc::new(tokio::sync::Semaphore::new(16)), + shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + sender, + }); + if api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connected(connection.clone()) + .is_some() + { + let _ = socket.close().await; + return; + } let registered = ExternalWorkdirServerFrame::current(ExternalWorkdirServerMessage::Registered { generation: connection.generation, @@ -25358,36 +23996,19 @@ async fn serve_external_workdir_provider( max_in_flight_operations: 16, }); if !send_external_workdir_server_frame(&mut socket, ®istered).await { - let stage_api = api.clone(); - let stage_grant = initial_grant.clone(); - let stage_connection = connection.clone(); - if let Err(error) = api - .store - .db_executor() - .run(move || { - stage_api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&stage_connection.grant_id, stage_connection.generation); - stage_api.store.update_external_workdir_grant_state( - &stage_grant.workspace_id, - &stage_grant.grant_id, - &stage_connection.provider_instance_id, - stage_connection.generation, - "offline", - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - }) - .await - { - // Rejected admission must not leave a dead socket discoverable. - // The generation-bound in-memory fence is safe even for an unknown - // DB outcome; it neither guesses nor retries the durable mutation. - api.external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); + let _ = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + if let Err(error) = api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &connection.provider_instance_id, + connection.generation, + "offline", + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) { tracing::error!( grant_id = %initial_grant.grant_id, generation = connection.generation, @@ -25537,81 +24158,55 @@ async fn serve_external_workdir_provider( } }; pending.clear(); - let stage_api = api.clone(); - let stage_grant = initial_grant.clone(); - let stage_connection = connection.clone(); - let terminal_update = api - .store - .db_executor() - .run(move || -> Result<_> { - let api = stage_api; - let initial_grant = stage_grant; - let connection = stage_connection; - let _ = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - let state_updated = match api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &connection.provider_instance_id, - connection.generation, - terminal_status, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { - Ok(updated) => updated, - Err(error) => { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to persist External Workdir provider terminal state" - ); - false - } - }; - if state_updated - && let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_disconnected", - &initial_grant.grant_id, - None, - terminal_status, - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to audit External Workdir provider terminal state" - ); - } - Ok(state_updated) - }) - .await; - let state_updated = match terminal_update { + let _ = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + let state_updated = match api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &connection.provider_instance_id, + connection.generation, + terminal_status, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) { Ok(updated) => updated, Err(error) => { - api.external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - tracing::error!(grant_id = %initial_grant.grant_id, - generation = connection.generation, status = terminal_status, %error, - "External Workdir provider terminal stage failed"); + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to persist External Workdir provider terminal state" + ); false } }; + if state_updated + && let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_disconnected", + &initial_grant.grant_id, + None, + terminal_status, + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + { + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to audit External Workdir provider terminal state" + ); + } if state_updated && matches!(terminal_status, "expired" | "revoked") && let Err(error) = @@ -25631,27 +24226,17 @@ async fn scoped_get_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let stage_api = api.clone(); - let stage_workspace_id = workspace_id.clone(); - let stage_grant_id = grant_id.clone(); + validate_workspace_scope(&api, &workspace_id)?; let mut grant = api .store - .db_executor() - .run(move || -> ApiResult<_> { - validate_workspace_scope(&stage_api, &stage_workspace_id)?; - let grant = stage_api - .store - .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir grant inspection requires the grant creator".to_string(), - ) - .into()); - } - Ok(grant) - }) - .await?; + .get_external_workdir_grant(&workspace_id, &grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir grant inspection requires the grant creator".to_string(), + ) + .into()); + } let expired = !matches!(grant.status.as_str(), "revoked" | "expired") && external_workdir_grant_is_expired(&grant)?; if expired { @@ -25678,57 +24263,44 @@ async fn scoped_revoke_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let stage_api = api.clone(); - let stage_workspace_id = workspace_id.clone(); - let stage_grant_id = grant_id.clone(); - let grant = api + validate_workspace_scope(&api, &workspace_id)?; + let mut grant = api .store - .db_executor() - .run(move || -> ApiResult<_> { - let api = stage_api; - let workspace_id = stage_workspace_id; - let grant_id = stage_grant_id; - validate_workspace_scope(&api, &workspace_id)?; - let mut grant = api - .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir revoke requires the grant creator".to_string(), - ) - .into()); - } - let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !matches!(grant.status.as_str(), "revoked" | "expired") { - api.store.update_external_workdir_grant_state( - &workspace_id, - &grant_id, - &grant.provider_instance_id, - grant.generation, - "revoked", - &updated_at, - )?; - grant.status = "revoked".to_string(); - grant.updated_at = updated_at; - api.store.record_external_workdir_audit( - &workspace_id, - "account", - &actor.account_id, - "external_workdir_grant_revoked", - &grant_id, - None, - "revoked", - &format!( - "workdir_id={} generation={}", - grant.workdir_id, grant.generation - ), - &grant.updated_at, - )?; - } - Ok(grant) - }) - .await?; + .get_external_workdir_grant(&workspace_id, &grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir revoke requires the grant creator".to_string(), + ) + .into()); + } + let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !matches!(grant.status.as_str(), "revoked" | "expired") { + api.store.update_external_workdir_grant_state( + &workspace_id, + &grant_id, + &grant.provider_instance_id, + grant.generation, + "revoked", + &updated_at, + )?; + grant.status = "revoked".to_string(); + grant.updated_at = updated_at; + api.store.record_external_workdir_audit( + &workspace_id, + "account", + &actor.account_id, + "external_workdir_grant_revoked", + &grant_id, + None, + "revoked", + &format!( + "workdir_id={} generation={}", + grant.workdir_id, grant.generation + ), + &grant.updated_at, + )?; + } // Cleanup is deliberately retried for terminal grants. A prior revoke may // have committed its fence before a Runtime/session projection failure. let cleanup = cleanup_external_workdir_attachments(&api, &grant.workdir_id).await; @@ -25753,88 +24325,70 @@ async fn cleanup_external_workdir_attachments( async fn cleanup_external_workdir_attachments_with_lock( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, + already_locked_worker: Option<&RuntimeWorkerRef>, ) -> ApiResult<()> { - let stage_api = api.clone(); - let stage_workdir_id = workdir_id.to_string(); - let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); - let grant_id = api + let workdir = api .store - .db_executor() - .run(move || -> Result<_> { - let _session_guard = stage_guard; - let workdir = stage_api - .store - .get_workdir_registry(&stage_api.config.workspace_id, &stage_workdir_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; - let grant_id = workdir - .source - .external_grant_id() - .ok_or_else(|| { - Error::InvalidInput("Workdir is not backed by an External grant".to_string()) - })? - .to_string(); - stage_api.store.update_external_workdir_cleanup_state( - &stage_api.config.workspace_id, - &grant_id, - "pending", - None, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; - Ok(grant_id) - }) - .await?; + .get_workdir_registry(&api.config.workspace_id, workdir_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; + let grant_id = workdir.source.external_grant_id().ok_or_else(|| { + Error::InvalidInput("Workdir is not backed by an External grant".to_string()) + })?; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + api.store.update_external_workdir_cleanup_state( + &api.config.workspace_id, + grant_id, + "pending", + None, + &now, + )?; - let release = - release_external_workdir_attachments(api, workdir_id, already_locked_worker).await; - let stage_api = api.clone(); - let stage_workdir_id = workdir_id.to_string(); - let succeeded = release.is_ok(); - let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); - api.store - .db_executor() - .run(move || -> Result<()> { - let _session_guard = stage_guard; - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let (state, category, event, outcome, detail) = if succeeded { - ( - "completed", - None, - "external_workdir_cleanup_completed", - "completed", - format!("workdir_id={stage_workdir_id}"), - ) - } else { - ( - "retry_required", - Some("attachment_release_failed"), - "external_workdir_cleanup_attention_required", - "attention_required", - format!("workdir_id={stage_workdir_id} category=attachment_release_failed"), - ) - }; - stage_api.store.update_external_workdir_cleanup_state( - &stage_api.config.workspace_id, - &grant_id, - state, - category, - &now, + match release_external_workdir_attachments(api, workdir_id, already_locked_worker).await { + Ok(()) => { + let completed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + api.store.update_external_workdir_cleanup_state( + &api.config.workspace_id, + grant_id, + "completed", + None, + &completed_at, )?; - stage_api.store.record_external_workdir_audit( - &stage_api.config.workspace_id, + api.store.record_external_workdir_audit( + &api.config.workspace_id, "system", "workspace-backend", - event, - &grant_id, + "external_workdir_cleanup_completed", + grant_id, None, - outcome, - &detail, - &now, + "completed", + &format!("workdir_id={workdir_id}"), + &completed_at, )?; Ok(()) - }) - .await?; - release + } + Err(error) => { + let failed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + api.store.update_external_workdir_cleanup_state( + &api.config.workspace_id, + grant_id, + "retry_required", + Some("attachment_release_failed"), + &failed_at, + )?; + api.store.record_external_workdir_audit( + &api.config.workspace_id, + "system", + "workspace-backend", + "external_workdir_cleanup_attention_required", + grant_id, + None, + "attention_required", + &format!("workdir_id={workdir_id} category=attachment_release_failed"), + &failed_at, + )?; + Err(error) + } + } } #[cfg(test)] @@ -25895,28 +24449,17 @@ async fn release_external_workdir_attachment_locked( api: &WorkspaceApi, workdir_id: &str, link: &WorkerWorkdirLinkRecord, - session_guard: &Arc>, ) -> ApiResult<()> { if link.unlinked_at.is_none() { close_current_worker_attachment_session_locked(api, &link.worker).await?; close_current_worker_command_sessions_for_alias_locked(api, &link.worker, &link.alias) .await?; - let stage_api = api.clone(); - let stage_worker = link.worker.clone(); - let stage_workdir_id = workdir_id.to_string(); - let stage_guard = session_guard.clone(); - api.store - .db_executor() - .run(move || { - let _session_guard = stage_guard; - stage_api.store.detach_worker_workdir( - &stage_api.config.workspace_id, - &stage_worker, - Some(&stage_workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - }) - .await?; + api.store.detach_worker_workdir( + &api.config.workspace_id, + &link.worker, + Some(workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; #[cfg(test)] if take_external_workdir_cleanup_failure(workdir_id) { return Err(Error::RuntimeOperationFailed { @@ -25931,10 +24474,9 @@ async fn release_external_workdir_attachment_locked( // have committed detach before Runtime persistence or projection failed. #[cfg(test)] record_external_workdir_cleanup_reconciliation(workdir_id); - sync_runtime_worker_workdir_attachments_locked(api, &link.worker, session_guard).await?; + sync_runtime_worker_workdir_attachments(api, &link.worker)?; api.worker_projection - .refresh_async(&link.worker) - .await + .refresh(&link.worker) .map_err(ApiError::from)?; Ok(()) } @@ -25942,34 +24484,23 @@ async fn release_external_workdir_attachment_locked( async fn release_external_workdir_attachments( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, + already_locked_worker: Option<&RuntimeWorkerRef>, ) -> ApiResult<()> { - let stage_api = api.clone(); - let stage_workdir_id = workdir_id.to_string(); let Some(link) = api .store - .db_executor() - .run(move || { - stage_api - .store - .latest_workdir_worker_link(&stage_api.config.workspace_id, &stage_workdir_id) - }) - .await? + .latest_workdir_worker_link(&api.config.workspace_id, workdir_id)? else { return Ok(()); }; // The newest link is the only possible active attachment and remains the // durable reconciliation target if detach committed before Runtime update. // Missing/ambiguous Runtime state remains a retryable cleanup failure. - if let Some((worker, guard)) = - already_locked_worker.filter(|(worker, _)| *worker == &link.worker) - { - debug_assert_eq!(worker, &link.worker); - release_external_workdir_attachment_locked(api, workdir_id, &link, guard).await?; + if already_locked_worker == Some(&link.worker) { + release_external_workdir_attachment_locked(api, workdir_id, &link).await?; } else { let lock = current_worker_session_lock(api, &link.worker); - let guard = Arc::new(lock.lock_owned().await); - release_external_workdir_attachment_locked(api, workdir_id, &link, &guard).await?; + let _guard = lock.lock().await; + release_external_workdir_attachment_locked(api, workdir_id, &link).await?; } Ok(()) } @@ -25978,19 +24509,14 @@ async fn scoped_list_working_directories( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let (items, diagnostics) = working_directory_summaries(&api)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let (items, diagnostics) = working_directory_summaries(&api)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) } async fn scoped_create_working_directory( @@ -26005,47 +24531,40 @@ async fn scoped_working_directory_detail( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let record = api - .store - .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "working_directory_not_found".to_string(), - message: format!("Unknown Workdir `{}`", path.working_directory_id), - })?; - if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { - let grant = api - .store - .get_workspace_config_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| { - Error::WorkspacePermissionDenied( - "Workspace config grant unavailable".into(), - ) - })?; - return Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: workspace_config::summary(&grant, &record), - diagnostics: vec![], - })); - } - if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { - working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) - } else { - let item = projected_workdir_summary_from_record(&api, &record)?; - Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: item.into(), - diagnostics: Vec::new(), - })) - } - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let record = api + .store + .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "working_directory_not_found".to_string(), + message: format!("Unknown Workdir `{}`", path.working_directory_id), + })?; + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| { + Error::WorkspacePermissionDenied("Workspace config grant unavailable".into()) + })?; + return Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: workspace_config::summary(&grant, &record), + diagnostics: vec![], + })); + } + if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { + working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) + } else { + let item = projected_workdir_summary_from_record(&api, &record)?; + Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: item.into(), + diagnostics: Vec::new(), + })) + } } async fn cleanup_working_directory( @@ -26055,18 +24574,13 @@ async fn cleanup_working_directory( request: WorkingDirectoryRemovalRequest, ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; - api.host_executors - .host()? - .run(move || { - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) - }) - .await + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) } fn registered_workdir_runtime_id( @@ -26138,49 +24652,12 @@ fn finish_rejected_workdir_create_operation( Ok(failure_code) } -async fn fail_workdir_create_stage( - api: &WorkspaceApi, - reserved: &WorkdirCreateOperationRecord, - request_fingerprint: &str, - failure: &str, - remove_registry: bool, -) -> Result { - let stage_api = api.clone(); - let reserved = reserved.clone(); - let request_fingerprint = request_fingerprint.to_string(); - let failure = failure.to_string(); - api.store - .db_executor() - .run(move || { - if remove_registry { - let _ = stage_api.store.delete_workdir_registry( - &reserved.workspace_id, - &reserved.working_directory_id, - ); - } - stage_api.config_store.finish_workdir_create_operation( - &reserved.workspace_id, - &reserved.operation_id, - &request_fingerprint, - false, - Some(&failure), - &now_registry_timestamp(), - ) - }) - .await -} - -fn prepare_workspace_workdir_create( +async fn create_workspace_working_directory( api: &WorkspaceApi, workspace_id: &str, route_runtime_id: Option<&str>, mut request: BrowserWorkingDirectoryCreateRequest, -) -> ApiResult<( - WorkingDirectoryRequest, - String, - String, - WorkdirCreateOperationRecord, -)> { +) -> ApiResult<(StatusCode, Json)> { validate_workspace_scope(api, workspace_id)?; request.display_name = request .display_name @@ -26339,111 +24816,55 @@ fn prepare_workspace_workdir_create( })? }; - Ok(( - working_directory_request, - operation_id, - request_fingerprint, - reserved, - )) -} - -async fn create_workspace_working_directory( - api: &WorkspaceApi, - workspace_id: &str, - route_runtime_id: Option<&str>, - request: BrowserWorkingDirectoryCreateRequest, -) -> ApiResult<(StatusCode, Json)> { - let stage_api = api.clone(); - let stage_workspace_id = workspace_id.to_string(); - let route_runtime_id = route_runtime_id.map(str::to_string); - let (mut working_directory_request, operation_id, request_fingerprint, reserved) = api - .store - .db_executor() - .run(move || { - prepare_workspace_workdir_create( - &stage_api, - &stage_workspace_id, - route_runtime_id.as_deref(), - request, - ) - }) - .await?; - if reserved.state == "succeeded" { - let stage_api = api.clone(); - return api - .host_executors - .host()? - .run(move || { - working_directory_detail_for_runtime( - stage_api, - &reserved.resolved_runtime_id, - &reserved.working_directory_id, - ) - .map(|Json(response)| { - ( - StatusCode::OK, - Json(BrowserWorkingDirectoryCreateResponse { - workspace_id: response.workspace_id, - runtime_id: response - .runtime_id - .expect("Repository Workdir detail has Runtime"), - item: response.item, - diagnostics: response.diagnostics, - http_status: StatusCode::OK.as_u16(), - }), - ) - }) - }) - .await; + return working_directory_detail_for_runtime( + api.clone(), + &reserved.resolved_runtime_id, + &reserved.working_directory_id, + ) + .map(|Json(response)| { + ( + StatusCode::OK, + Json(BrowserWorkingDirectoryCreateResponse { + workspace_id: response.workspace_id, + runtime_id: response + .runtime_id + .expect("Repository Workdir detail has Runtime"), + item: response.item, + diagnostics: response.diagnostics, + http_status: StatusCode::OK.as_u16(), + }), + ) + }); } let reserved = if reserved.state == "failed" { - let stage_api = api.clone(); - let stage_workspace_id = workspace_id.to_string(); - let stage_operation_id = operation_id.clone(); - let stage_fingerprint = request_fingerprint.clone(); - api.store - .db_executor() - .run(move || { - stage_api.config_store.begin_failed_workdir_create_retry( - &stage_workspace_id, - &stage_operation_id, - &stage_fingerprint, - &now_registry_timestamp(), - ) - }) - .await? + api.config_store.begin_failed_workdir_create_retry( + workspace_id, + &operation_id, + &request_fingerprint, + &now_registry_timestamp(), + )? } else { reserved }; - let stage_api = api.clone(); - let stage_runtime_id = reserved.resolved_runtime_id.clone(); let runtime = match api - .host_executors - .host()? - .run(move || { - Ok::<_, Error>( - stage_api - .runtime - .list_runtimes(usize::MAX) - .items - .into_iter() - .find(|runtime| runtime.runtime_id == stage_runtime_id), - ) - }) - .await? + .runtime + .list_runtimes(usize::MAX) + .items + .into_iter() + .find(|runtime| runtime.runtime_id == reserved.resolved_runtime_id) { Some(runtime) => runtime, None => { - fail_workdir_create_stage( - api, - &reserved, + api.config_store.finish_workdir_create_operation( + workspace_id, + &operation_id, &request_fingerprint, - "runtime_unavailable", false, - ) - .await?; + Some("runtime_unavailable"), + &now_registry_timestamp(), + )?; return Err(ApiError::with_diagnostics( Error::UnknownRuntime(reserved.resolved_runtime_id), vec![RuntimeDiagnostic { @@ -26455,14 +24876,14 @@ async fn create_workspace_working_directory( } }; if runtime.kind == "remote_http" && runtime.status != "connected" { - fail_workdir_create_stage( - api, - &reserved, + api.config_store.finish_workdir_create_operation( + workspace_id, + &operation_id, &request_fingerprint, - "runtime_unavailable", false, - ) - .await?; + Some("runtime_unavailable"), + &now_registry_timestamp(), + )?; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: reserved.resolved_runtime_id, @@ -26477,52 +24898,38 @@ async fn create_workspace_working_directory( )); } - let stage_api = api.clone(); - let stage_reserved = reserved.clone(); - let stage_fingerprint = request_fingerprint.clone(); - let (authorization, mut working_directory_request) = api - .store - .db_executor() - .run(move || { - let authorization = authorize_repository_materialization_operation( - &stage_api, - &stage_reserved, - &stage_fingerprint, - &mut working_directory_request, - ); - Ok::<_, Error>((authorization, working_directory_request)) - }) - .await?; - if let Err(error) = authorization { - fail_workdir_create_stage( - api, - &reserved, + if let Err(error) = authorize_repository_materialization_operation( + api, + &reserved, + &request_fingerprint, + &mut working_directory_request, + ) { + api.config_store.finish_workdir_create_operation( + workspace_id, + &operation_id, &request_fingerprint, - "working_directory_remote_repository_access_required", false, - ) - .await?; + Some("working_directory_remote_repository_access_required"), + &now_registry_timestamp(), + )?; return Err(error); } working_directory_request.backend_workdir_id = Some(reserved.working_directory_id.clone()); - let stage_api = api.clone(); - let stage_reserved = reserved.clone(); - let existing = match api - .host_executors - .host()? - .run(move || { - Ok::<_, Error>(stage_api.runtime.working_directory( - &stage_reserved.resolved_runtime_id, - &stage_reserved.working_directory_id, - )) - }) - .await? - { + let existing = match api.runtime.working_directory( + &reserved.resolved_runtime_id, + &reserved.working_directory_id, + ) { Ok(existing) => existing, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - fail_workdir_create_stage(api, &reserved, &request_fingerprint, &failure_code, false) - .await?; + api.config_store.finish_workdir_create_operation( + workspace_id, + &operation_id, + &request_fingerprint, + false, + Some(&failure_code), + &now_registry_timestamp(), + )?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -26536,31 +24943,24 @@ async fn create_workspace_working_directory( let result = if existing.working_directory.is_some() { existing } else { - let stage_api = api.clone(); - let stage_runtime_id = reserved.resolved_runtime_id.clone(); match api - .host_executors - .host()? - .run(move || { - Ok::<_, Error>( - stage_api - .runtime - .create_working_directory(&stage_runtime_id, working_directory_request), - ) - }) - .await? + .runtime + .create_working_directory(&reserved.resolved_runtime_id, working_directory_request) { Ok(result) => result, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - fail_workdir_create_stage( - api, - &reserved, + let _ = api + .store + .delete_workdir_registry(workspace_id, &reserved.working_directory_id); + api.config_store.finish_workdir_create_operation( + workspace_id, + &operation_id, &request_fingerprint, - &failure_code, - true, - ) - .await?; + false, + Some(&failure_code), + &now_registry_timestamp(), + )?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -26573,30 +24973,18 @@ async fn create_workspace_working_directory( } }; let Some(working_directory) = result.working_directory else { - let mut diagnostics = result.diagnostics; - let stage_api = api.clone(); - let stage_reserved = reserved.clone(); - let stage_fingerprint = request_fingerprint.clone(); - let (failure_code, returned_diagnostics) = api + let _ = api .store - .db_executor() - .run(move || -> Result<_> { - let _ = stage_api.store.delete_workdir_registry( - &stage_reserved.workspace_id, - &stage_reserved.working_directory_id, - ); - let failure_code = finish_rejected_workdir_create_operation( - &stage_api.config_store, - &stage_reserved.workspace_id, - &stage_reserved.operation_id, - &stage_fingerprint, - &diagnostics, - &now_registry_timestamp(), - )?; - Ok((failure_code, diagnostics)) - }) - .await?; - diagnostics = returned_diagnostics; + .delete_workdir_registry(workspace_id, &reserved.working_directory_id); + let mut diagnostics = result.diagnostics; + let failure_code = finish_rejected_workdir_create_operation( + &api.config_store, + workspace_id, + &operation_id, + &request_fingerprint, + &diagnostics, + &now_registry_timestamp(), + )?; if !diagnostics.iter().any(|diagnostic| { diagnostic.severity == HostDiagnosticSeverity::Error && diagnostic.code == failure_code }) { @@ -26618,29 +25006,21 @@ async fn create_workspace_working_directory( diagnostics, )); }; - let stage_api = api.clone(); - let stage_reserved = reserved.clone(); - let summary = api - .store - .db_executor() - .run(move || -> ApiResult<_> { - let record = workdir_record_from_summary( - &stage_api, - &stage_reserved.resolved_runtime_id, - &working_directory.summary, - ); - stage_api.store.upsert_workdir_registry(&record)?; - stage_api.config_store.finish_workdir_create_operation( - &stage_reserved.workspace_id, - &stage_reserved.operation_id, - &request_fingerprint, - true, - None, - &now_registry_timestamp(), - )?; - Ok(projected_workdir_summary_from_record(&stage_api, &record)?) - }) - .await?; + let record = workdir_record_from_summary( + api, + &reserved.resolved_runtime_id, + &working_directory.summary, + ); + api.store.upsert_workdir_registry(&record)?; + api.config_store.finish_workdir_create_operation( + workspace_id, + &operation_id, + &request_fingerprint, + true, + None, + &now_registry_timestamp(), + )?; + let summary = projected_workdir_summary_from_record(api, &record)?; Ok(( StatusCode::CREATED, Json(BrowserWorkingDirectoryCreateResponse { @@ -27220,7 +25600,7 @@ async fn set_worker_retention( runtime_worker_id: String, pinned: bool, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &runtime_worker_id).await?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &runtime_worker_id)?; Ok(Json( worker .set_pinned(&WorkerOperationContext::Backend, pinned) @@ -27484,13 +25864,7 @@ async fn execute_runtime_cleanup_with_context( } context.authorize_workspace(api).await?; - let plan_api = api.clone(); - let plan_runtime = runtime_id.to_owned(); - let plan = api - .host_executors - .host()? - .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) - .await?; + let plan = build_runtime_cleanup_plan(api, runtime_id)?; // Rebuild the actual candidate set immediately before executing. The digest // binds targets, dependencies and blockers; per-target authority is checked again below. if request.expected_plan_digest != plan.digest { @@ -27513,36 +25887,20 @@ async fn execute_runtime_cleanup_with_context( candidate.runtime_id.clone(), candidate.runtime_worker_id.clone(), ); - let authority_api = api.clone(); - let target = worker.clone(); - let (claim, assignment) = api + if let Some(claim) = api .store - .db_executor() - .run(move || -> Result<_> { - Ok(( - authority_api - .store - .get_pending_ticket_assignment_operation_for_worker( - &authority_api.config.workspace_id, - &target, - )?, - authority_api - .store - .get_active_ticket_role_assignment_for_worker( - &authority_api.config.workspace_id, - &target, - )?, - )) - }) - .await?; - if let Some(claim) = claim { + .get_pending_ticket_assignment_operation_for_worker(&api.config.workspace_id, &worker)? + { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_claim_pending", &format!("Worker has pending Ticket claim {}", claim.operation_id), )); } - if let Some(assignment) = assignment { + if let Some(assignment) = api + .store + .get_active_ticket_role_assignment_for_worker(&api.config.workspace_id, &worker)? + { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_assigned", @@ -27568,8 +25926,7 @@ async fn execute_runtime_cleanup_with_context( )); } parse_runtime_worker_id_for_registry(&candidate.runtime_worker_id)?; - WorkspaceWorker::resolve_async(api, &worker.runtime_id, &worker.worker_id) - .await? + WorkspaceWorker::resolve(api, &worker.runtime_id, &worker.worker_id)? .delete_from_plan(context, candidate) .await?; api.workdir_session_locks @@ -27608,21 +25965,12 @@ async fn execute_runtime_cleanup_with_context( }); } CleanupTargetKind::WorkdirCleanCleanup | CleanupTargetKind::WorkdirRecordDelete => { - let removal_api = api.clone(); - let removal_candidate = candidate.clone(); - let requester = format!("runtime-cleanup:{runtime_id}"); - let removal = api - .host_executors - .host()? - .run(move || { - execute_workdir_removal( - &removal_api, - removal_candidate.workdir_id.as_str(), - &requester, - &format!("cleanup target {}", removal_candidate.target_id), - ) - }) - .await?; + let removal = execute_workdir_removal( + api, + candidate.workdir_id.as_str(), + &format!("runtime-cleanup:{runtime_id}"), + &format!("cleanup target {}", candidate.target_id), + )?; let (status, message) = match removal.disposition { WorkingDirectoryRemovalDisposition::Removed => ( "deleted", @@ -27668,13 +26016,7 @@ async fn execute_runtime_cleanup_with_context( )); } - let plan_api = api.clone(); - let plan_runtime = runtime_id.to_owned(); - let plan_after = api - .host_executors - .host()? - .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) - .await?; + let plan_after = build_runtime_cleanup_plan(api, runtime_id)?; let executed_at = now_registry_timestamp(); Ok(RuntimeCleanupExecutionResponse { workspace_id: api.config.workspace_id.clone(), @@ -27947,9 +26289,7 @@ async fn scoped_restore_runtime_worker_with_context( request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = - WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) - .await?; + let resolved = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; let _authorization = resolved.authorize_operation(&context, "restore").await?; let runtime_id = resolved.identity().runtime_id.clone(); let worker_id = resolved.identity().worker_id.clone(); @@ -27982,14 +26322,9 @@ async fn scoped_runtime_cleanup_plan( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; - Ok(Json(plan)) - }) - .await + validate_workspace_scope(&api, &path.workspace_id)?; + let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; + Ok(Json(plan)) } #[cfg(test)] @@ -28034,65 +26369,56 @@ async fn scoped_create_attachment_upload_grant( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - resolve_workspace_worker_reference( - &api, - &path.worker.runtime_id, - &path.worker.worker_id, - )?; - let now = attachment_upload_now_ms()?; - let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); - let upload_id = match query.upload_id { - Some(upload_id) => Uuid::parse_str(&upload_id) - .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? - .to_string(), - None => Uuid::now_v7().to_string(), - }; - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - grants.retain(|_, grant| grant.expires_at_ms >= now); - if let Some(existing) = grants.get(&upload_id) { - if existing.workspace_id == path.workspace_id - && existing.runtime_id == path.worker.runtime_id - && existing.worker_id == path.worker.worker_id - && existing.account_id == actor.account_id - && existing.file_name == query.file_name - && existing.media_type == query.media_type - { - return Ok(Json(AttachmentUploadGrantResponse { - upload_id, - expires_at_ms: existing.expires_at_ms, - })); - } - return Err(Error::RepositoryConflict( - "attachment upload id was reused with different intent".into(), - ) - .into()); - } - grants.insert( - upload_id.clone(), - AttachmentUploadGrant { - workspace_id: path.workspace_id, - runtime_id: path.worker.runtime_id, - worker_id: path.worker.worker_id, - account_id: actor.account_id, - file_name: query.file_name, - media_type: query.media_type, - expires_at_ms, - state: AttachmentUploadGrantState::Pending, - }, - ); - Ok(Json(AttachmentUploadGrantResponse { + validate_workspace_scope(&api, &path.workspace_id)?; + resolve_workspace_worker_reference(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let now = attachment_upload_now_ms()?; + let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); + let upload_id = match query.upload_id { + Some(upload_id) => Uuid::parse_str(&upload_id) + .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? + .to_string(), + None => Uuid::now_v7().to_string(), + }; + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + grants.retain(|_, grant| grant.expires_at_ms >= now); + if let Some(existing) = grants.get(&upload_id) { + if existing.workspace_id == path.workspace_id + && existing.runtime_id == path.worker.runtime_id + && existing.worker_id == path.worker.worker_id + && existing.account_id == actor.account_id + && existing.file_name == query.file_name + && existing.media_type == query.media_type + { + return Ok(Json(AttachmentUploadGrantResponse { upload_id, - expires_at_ms, - })) - }) - .await + expires_at_ms: existing.expires_at_ms, + })); + } + return Err(Error::RepositoryConflict( + "attachment upload id was reused with different intent".into(), + ) + .into()); + } + grants.insert( + upload_id.clone(), + AttachmentUploadGrant { + workspace_id: path.workspace_id, + runtime_id: path.worker.runtime_id, + worker_id: path.worker.worker_id, + account_id: actor.account_id, + file_name: query.file_name, + media_type: query.media_type, + expires_at_ms, + state: AttachmentUploadGrantState::Pending, + }, + ); + Ok(Json(AttachmentUploadGrantResponse { + upload_id, + expires_at_ms, + })) } async fn upload_runtime_worker_file_contract( @@ -28177,55 +26503,49 @@ async fn scoped_cancel_attachment_upload( Extension(actor): Extension, AxumPath(path): AxumPath, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let completed = { - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - let grant = grants.get_mut(&path.upload_id).ok_or_else(|| { - Error::InvalidInput("attachment upload grant is unknown".into()) - })?; - if grant.workspace_id != path.workspace_id - || grant.runtime_id != path.runtime_id - || grant.worker_id != path.worker_id - || grant.account_id != actor.account_id - { - return Err(Error::WorkspacePermissionDenied( - "attachment upload grant scope mismatch".into(), - ) - .into()); - } - match &grant.state { - AttachmentUploadGrantState::Pending => { - grants.remove(&path.upload_id); - None - } - AttachmentUploadGrantState::Uploading => { - grant.state = AttachmentUploadGrantState::Cancelled; - None - } - AttachmentUploadGrantState::Cancelled => None, - AttachmentUploadGrantState::Completed(file) => { - let file = file.clone(); - grants.remove(&path.upload_id); - Some(file) - } - } - }; - if let Some(file) = completed { - let worker = - resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &file.artifact_id) - .map_err(|error| error.into_error())?; + validate_workspace_scope(&api, &path.workspace_id)?; + let completed = { + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + let grant = grants + .get_mut(&path.upload_id) + .ok_or_else(|| Error::InvalidInput("attachment upload grant is unknown".into()))?; + if grant.workspace_id != path.workspace_id + || grant.runtime_id != path.runtime_id + || grant.worker_id != path.worker_id + || grant.account_id != actor.account_id + { + return Err(Error::WorkspacePermissionDenied( + "attachment upload grant scope mismatch".into(), + ) + .into()); + } + match &grant.state { + AttachmentUploadGrantState::Pending => { + grants.remove(&path.upload_id); + None } - Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) - }) - .await + AttachmentUploadGrantState::Uploading => { + grant.state = AttachmentUploadGrantState::Cancelled; + None + } + AttachmentUploadGrantState::Cancelled => None, + AttachmentUploadGrantState::Completed(file) => { + let file = file.clone(); + grants.remove(&path.upload_id); + Some(file) + } + } + }; + if let Some(file) = completed { + let worker = resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &file.artifact_id) + .map_err(|error| error.into_error())?; + } + Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) } async fn scoped_delete_runtime_worker_uploaded_file( @@ -28302,13 +26622,8 @@ async fn post_auth_bootstrap_user( State(api): State, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - Ok(Json(AuthUserResponse { user })) - }) - .await + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + Ok(Json(AuthUserResponse { user })) } async fn post_passkey_registration_options( @@ -28316,53 +26631,48 @@ async fn post_passkey_registration_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let exclude_credentials = passkeys_for_user(&api, &user.user_id)? - .into_iter() - .map(|passkey| passkey.cred_id().clone()) - .collect(); - let user_unique_id = Uuid::now_v7(); - let (public_key, state) = webauthn - .start_passkey_registration( - user_unique_id, - &user.handle, - &user.display_name, - Some(exclude_credentials), - ) - .map_err(|error| { - auth_error("webauthn_registration_options_failed", &error.to_string()) - })?; - let challenge_id = new_id("webauthn-registration"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_registration".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some(serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyRegistrationOptionsResponse { - challenge_id, - public_key, - })) - }) - .await + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let exclude_credentials = passkeys_for_user(&api, &user.user_id)? + .into_iter() + .map(|passkey| passkey.cred_id().clone()) + .collect(); + let user_unique_id = Uuid::now_v7(); + let (public_key, state) = webauthn + .start_passkey_registration( + user_unique_id, + &user.handle, + &user.display_name, + Some(exclude_credentials), + ) + .map_err(|error| auth_error("webauthn_registration_options_failed", &error.to_string()))?; + let challenge_id = new_id("webauthn-registration"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_registration".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some( + serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?, + ), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyRegistrationOptionsResponse { + challenge_id, + public_key, + })) } fn complete_passkey_registration( @@ -28439,53 +26749,50 @@ async fn post_passkey_login_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let user = match request.handle.as_deref() { - Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, - None => api.store.any_user()?, - } - .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; - let passkeys = passkeys_for_user(&api, &user.user_id)?; - if passkeys.is_empty() { - return Err(auth_error( - "passkey_not_registered", - "user has no registered passkey credentials", - ) - .into()); - } - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let (public_key, state) = webauthn - .start_passkey_authentication(&passkeys) - .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; - let challenge_id = new_id("webauthn-login"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_login".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some(serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyLoginOptionsResponse { - challenge_id, - public_key, - })) - }) - .await + let user = match request.handle.as_deref() { + Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, + None => api.store.any_user()?, + } + .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; + let passkeys = passkeys_for_user(&api, &user.user_id)?; + if passkeys.is_empty() { + return Err(auth_error( + "passkey_not_registered", + "user has no registered passkey credentials", + ) + .into()); + } + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let (public_key, state) = webauthn + .start_passkey_authentication(&passkeys) + .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; + let challenge_id = new_id("webauthn-login"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_login".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some( + serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?, + ), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyLoginOptionsResponse { + challenge_id, + public_key, + })) } fn complete_passkey_login( @@ -28603,53 +26910,48 @@ async fn post_device_login_start( State(api): State, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; + const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; - let auth = auth_public_config(&api.config); - let verification_uri = format!( - "{}/login/device", - auth.public_base_url.trim_end_matches('/') - ); - for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { - let device_code = mint_secret("yoi_device"); - let user_code = new_user_code(); - let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); - let flow = DeviceLoginFlowRecord { - device_code: device_code.clone(), - user_code: user_code.clone(), - verification_uri: verification_uri.clone(), - client_name: request.client_name.clone(), - user_id: None, - api_token_id: None, - issued_access_token: None, - created_at: crate::auth::now_rfc3339(), - expires_at: rfc3339_after(Duration::minutes(10)), - approved_at: None, - consumed_at: None, - }; - if !api.store.try_create_device_login_flow(&flow)? { - continue; - } - return Ok(Json(DeviceLoginStartResponse { - device_code, - user_code, - verification_uri, - verification_uri_complete, - expires_in: 600, - interval: 5, - })); - } + let auth = auth_public_config(&api.config); + let verification_uri = format!( + "{}/login/device", + auth.public_base_url.trim_end_matches('/') + ); + for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { + let device_code = mint_secret("yoi_device"); + let user_code = new_user_code(); + let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); + let flow = DeviceLoginFlowRecord { + device_code: device_code.clone(), + user_code: user_code.clone(), + verification_uri: verification_uri.clone(), + client_name: request.client_name.clone(), + user_id: None, + api_token_id: None, + issued_access_token: None, + created_at: crate::auth::now_rfc3339(), + expires_at: rfc3339_after(Duration::minutes(10)), + approved_at: None, + consumed_at: None, + }; + if !api.store.try_create_device_login_flow(&flow)? { + continue; + } + return Ok(Json(DeviceLoginStartResponse { + device_code, + user_code, + verification_uri, + verification_uri_complete, + expires_in: 600, + interval: 5, + })); + } - Err(auth_error( - "device_login_code_allocation_failed", - "could not allocate a unique device login user code", - ) - .into()) - }) - .await + Err(auth_error( + "device_login_code_allocation_failed", + "could not allocate a unique device login user code", + ) + .into()) } fn approve_device_login( @@ -28715,48 +27017,41 @@ async fn post_device_login_poll( State(api): State, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let Some(flow) = api - .store - .get_device_login_flow_by_device_code(&request.device_code)? - else { - return Err( - auth_error("unknown_device_login", "device login flow does not exist").into(), - ); - }; - if is_expired(&flow.expires_at) { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Expired, - access_token: None, - token_type: None, - })); - } - if flow.approved_at.is_none() { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Pending, - access_token: None, - token_type: None, - })); - } - let Some(consumed) = api - .store - .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? - else { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Consumed, - access_token: None, - token_type: None, - })); - }; - Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Approved, - access_token: consumed.issued_access_token, - token_type: Some(DeviceAccessTokenType::Bearer), - })) - }) - .await + let Some(flow) = api + .store + .get_device_login_flow_by_device_code(&request.device_code)? + else { + return Err(auth_error("unknown_device_login", "device login flow does not exist").into()); + }; + if is_expired(&flow.expires_at) { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Expired, + access_token: None, + token_type: None, + })); + } + if flow.approved_at.is_none() { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Pending, + access_token: None, + token_type: None, + })); + } + let Some(consumed) = api + .store + .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? + else { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Consumed, + access_token: None, + token_type: None, + })); + }; + Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Approved, + access_token: consumed.issued_access_token, + token_type: Some(DeviceAccessTokenType::Bearer), + })) } fn logout_browser_session( @@ -28950,7 +27245,7 @@ async fn resolve_actor( headers: &HeaderMap, ) -> ApiResult> { let cookie_name = auth_public_config(&api.config).cookie_name; - Ok(resolve_request_actor(api.store.clone(), headers, &cookie_name).await?) + Ok(resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await?) } async fn require_actor(api: &ServerAuthApi, headers: &HeaderMap) -> ApiResult { @@ -29042,130 +27337,103 @@ async fn list_tickets( State(api): State, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let limit = query.limit.unwrap_or(30).clamp(1, 100); - let states = query - .states - .as_deref() - .map(|states| { - states - .split(',') - .filter(|state| !state.is_empty()) - .map(str::to_string) - .collect::>() - }) - .unwrap_or_default(); - let crate::records::TicketSummaryPage { - items, - page, - invalid_records, - record_authority, - } = api - .authority - .list_ticket_page(crate::records::TicketListProjectionRequest { - states, - limit: Some(limit), - cursor: query.cursor, - })?; - Ok(Json(crate::records::TicketListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - page, - invalid_records, - record_authority, - })) + let limit = query.limit.unwrap_or(30).clamp(1, 100); + let states = query + .states + .as_deref() + .map(|states| { + states + .split(',') + .filter(|state| !state.is_empty()) + .map(str::to_string) + .collect::>() }) - .await + .unwrap_or_default(); + let crate::records::TicketSummaryPage { + items, + page, + invalid_records, + record_authority, + } = api + .authority + .list_ticket_page(crate::records::TicketListProjectionRequest { + states, + limit: Some(limit), + cursor: query.cursor, + })?; + Ok(Json(crate::records::TicketListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + page, + invalid_records, + record_authority, + })) } async fn get_ticket( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || Ok(Json(api.authority.ticket(&id)?))) - .await + Ok(Json(api.authority.ticket(&id)?)) } async fn list_objectives( State(api): State, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let limit = query.limit.unwrap_or(api.config.max_records).min(1000); - let ProjectRecordList { - items, - invalid_records, - record_authority, - } = api.authority.list_objectives(limit)?; - Ok(Json(server_api::ObjectiveListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - invalid_records, - record_authority, - })) - }) - .await + let limit = query.limit.unwrap_or(api.config.max_records).min(1000); + let ProjectRecordList { + items, + invalid_records, + record_authority, + } = api.authority.list_objectives(limit)?; + Ok(Json(server_api::ObjectiveListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + invalid_records, + record_authority, + })) } async fn get_objective( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || Ok(Json(api.authority.objective(&id)?))) - .await + Ok(Json(api.authority.objective(&id)?)) } async fn list_repositories( State(api): State, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let RepositoryListProjection { items, diagnostics } = - api.live_repository_reader()?.list(); - Ok(Json(RepositoryListResponse { - workspace_id: api.config.workspace_id, - items, - source: "workspace-control-plane".to_string(), - diagnostics, - })) - }) - .await + let RepositoryListProjection { items, diagnostics } = api.live_repository_reader()?.list(); + Ok(Json(RepositoryListResponse { + workspace_id: api.config.workspace_id, + items, + source: "workspace-control-plane".to_string(), + diagnostics, + })) } async fn repository_detail( State(api): State, AxumPath(repository_key): AxumPath, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let item = repository_lookup( - api.live_repository_reader()? - .summary_by_key(&repository_key), - )?; - Ok(Json(RepositoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - item, - source: "workspace-control-plane".to_string(), - })) - }) - .await + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let item = repository_lookup( + api.live_repository_reader()? + .summary_by_key(&repository_key), + )?; + Ok(Json(RepositoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + item, + source: "workspace-control-plane".to_string(), + })) } async fn repository_log( @@ -29173,81 +27441,63 @@ async fn repository_log( AxumPath(repository_key): AxumPath, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let RepositoryLogRead { - repository_key, - default_selector, - limit, - commits, - diagnostics, - } = repository_lookup( - api.live_repository_reader()? - .recent_log(&repository_key, query.limit), - )?; - Ok(Json(RepositoryLogResponse { - workspace_id: api.config.workspace_id, - repository_key, - default_selector, - limit, - items: commits, - diagnostics, - })) - }) - .await + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let RepositoryLogRead { + repository_key, + default_selector, + limit, + commits, + diagnostics, + } = repository_lookup( + api.live_repository_reader()? + .recent_log(&repository_key, query.limit), + )?; + Ok(Json(RepositoryLogResponse { + workspace_id: api.config.workspace_id, + repository_key, + default_selector, + limit, + items: commits, + diagnostics, + })) } async fn list_hosts( State(api): State, ) -> ApiResult>> { - api.host_executors - .host()? - .run(move || { - let limit = api.config.max_records.min(200); - let runtime_hosts = api.runtime.list_hosts(limit); - Ok(Json(InternalRuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtime_hosts.items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_hosts.diagnostics, - })) - }) - .await + let limit = api.config.max_records.min(200); + let runtime_hosts = api.runtime.list_hosts(limit); + Ok(Json(InternalRuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtime_hosts.items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_hosts.diagnostics, + })) } async fn list_runtimes( State(api): State, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let limit = api.config.max_records.min(200); - let runtimes = api.runtime.list_runtimes(limit); - Ok(Json(server_api::RuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtimes.items.into_iter().map(Into::into).collect(), - source: "worker_runtime_registry".to_string(), - diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), - })) - }) - .await + let limit = api.config.max_records.min(200); + let runtimes = api.runtime.list_runtimes(limit); + Ok(Json(server_api::RuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtimes.items.into_iter().map(Into::into).collect(), + source: "worker_runtime_registry".to_string(), + diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), + })) } async fn list_workers( State(api): State, ) -> ApiResult> { - api.store - .db_executor() - .run(move || workers_response(api).map(Json)) - .await + workers_response(api).map(Json) } async fn create_remote_runtime( @@ -29419,19 +27669,6 @@ fn runtime_removal_config_guard( async fn execute_runtime_removal( api: &WorkspaceApi, operation: RuntimeRemovalOperation, -) -> ApiResult { - let api = api.clone(); - // Retain registration/commit recovery as one owned lifecycle workflow; - // SQL transaction boundaries and commit-before-Runtime order are unchanged. - run_host_workflow(api.host_executors.host()?, async move { - execute_runtime_removal_on_host(&api, operation).await - }) - .await -} - -async fn execute_runtime_removal_on_host( - api: &WorkspaceApi, - operation: RuntimeRemovalOperation, ) -> ApiResult { if operation.state == RuntimeRemovalOperationState::Succeeded { return Ok(operation); @@ -29634,16 +27871,9 @@ async fn remove_remote_runtime( return Ok(Json(runtime_removal_response(operation))); } - let config_api = api.clone(); let config_state = api - .store - .db_executor() - .run(move || { - config_api - .config_store - .load_workspace_config(&config_api.config.workspace_id) - }) - .await? + .config_store + .load_workspace_config(&api.config.workspace_id)? .ok_or_else(|| { Error::RegistryInconsistency(format!( "Workspace {} has no active configuration", @@ -29724,10 +27954,7 @@ async fn test_runtime_connection( async fn get_worker_launch_options( State(api): State, ) -> ApiResult> { - api.store - .db_executor() - .run(move || Ok(Json(worker_launch_options_response(&api)?))) - .await + Ok(Json(worker_launch_options_response(&api)?)) } fn working_directory_request_from_repository( @@ -29906,27 +28133,6 @@ async fn create_workspace_worker_authenticated( request: CreateWorkspaceWorkerRequest, resolved_control_operation: Option, context: Option<&server_api::ServerRequestContext>, -) -> ApiResult> { - let context = context.cloned(); - run_host_workflow(api.host_executors.host()?, async move { - create_workspace_worker_authenticated_on_host( - api, - headers, - request, - resolved_control_operation, - context.as_ref(), - ) - .await - }) - .await -} - -async fn create_workspace_worker_authenticated_on_host( - api: WorkspaceApi, - headers: HeaderMap, - request: CreateWorkspaceWorkerRequest, - resolved_control_operation: Option, - context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let claim_lock = request .ticket_assignment @@ -30601,44 +28807,30 @@ fn backend_resource_error_status(error: &BackendResourceError) -> StatusCode { async fn get_companion_status( State(api): State, ) -> ApiResult> { - api.store - .db_executor() - .run(move || Ok(Json(api.companion.status()))) - .await + Ok(Json(api.companion.status())) } async fn get_companion_transcript( State(api): State, Query(query): Query, ) -> ApiResult> { - api.store - .db_executor() - .run(move || { - let limit = query.limit.unwrap_or(api.config.max_records).min(200); - let start = query.start.unwrap_or(0); - Ok(Json(api.companion.transcript(start, limit))) - }) - .await + let limit = query.limit.unwrap_or(api.config.max_records).min(200); + let start = query.start.unwrap_or(0); + Ok(Json(api.companion.transcript(start, limit))) } async fn post_companion_message( State(api): State, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || Ok(Json(api.companion.send_message(request)))) - .await + Ok(Json(api.companion.send_message(request))) } async fn post_companion_cancel( State(api): State, Json(request): Json, ) -> ApiResult> { - api.store - .db_executor() - .run(move || Ok(Json(api.companion.cancel(request)))) - .await + Ok(Json(api.companion.cancel(request))) } fn runtime_contract_restore( @@ -30737,30 +28929,25 @@ async fn get_runtime_worker( State(api): State, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let worker = api - .runtime - .worker(&worker_ref) - .map_err(|err| err.into_error())?; - let record = sync_worker_observation(&api, &worker)?; - let links = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; - let workdirs = api - .store - .list_workdir_registry(&api.config.workspace_id, 500)?; - let updated_at = record.updated_at.clone(); - let worker = - project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; - Ok(Json(server_api::RuntimeWorkerShowResponse { - worker, - updated_at, - })) - }) - .await + let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let worker = api + .runtime + .worker(&worker_ref) + .map_err(|err| err.into_error())?; + let record = sync_worker_observation(&api, &worker)?; + let links = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; + let workdirs = api + .store + .list_workdir_registry(&api.config.workspace_id, 500)?; + let updated_at = record.updated_at.clone(); + let worker = + project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; + Ok(Json(server_api::RuntimeWorkerShowResponse { + worker, + updated_at, + })) } async fn restore_runtime_worker_with_context( @@ -30770,7 +28957,7 @@ async fn restore_runtime_worker_with_context( context: WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; let mut result = worker.restore(&context, request).await?; let projected_worker = if let Some(worker) = result.worker.as_ref() { // A replayed terminal receipt is historical evidence, not a new Runtime observation. @@ -30835,38 +29022,33 @@ async fn list_runtime_workers( AxumPath(runtime_id): AxumPath, Query(query): Query, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let limit = api.config.max_records.min(200); - let (runtime_workers, source) = match query.status { - Some(RuntimeWorkersStatusFilter::Stopped) => ( - api.runtime - .list_stopped_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry_stopped", - ), - None => ( - api.runtime - .list_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry", - ), - }; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::RuntimeWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: source.to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) - }) - .await + let limit = api.config.max_records.min(200); + let (runtime_workers, source) = match query.status { + Some(RuntimeWorkersStatusFilter::Stopped) => ( + api.runtime + .list_stopped_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry_stopped", + ), + None => ( + api.runtime + .list_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry", + ), + }; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::RuntimeWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: source.to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -31267,25 +29449,6 @@ async fn create_runtime_worker_authenticated( AxumPath(runtime_id): AxumPath, Json(request): Json, context: Option<&server_api::ServerRequestContext>, -) -> ApiResult> { - let context = context.cloned(); - run_host_workflow(api.host_executors.host()?, async move { - create_runtime_worker_authenticated_on_host( - State(api), - AxumPath(runtime_id), - Json(request), - context.as_ref(), - ) - .await - }) - .await -} - -async fn create_runtime_worker_authenticated_on_host( - State(api): State, - AxumPath(runtime_id): AxumPath, - Json(request): Json, - context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let mut request = worker_spawn_request_from_api(request)?; let headers = context @@ -31487,22 +29650,17 @@ async fn sync_runtime_config_bundle( AxumPath(runtime_id): AxumPath, Json(request): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let bundle = serde_json::from_value::(request.bundle) - .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; - let result = api - .runtime - .sync_config_bundle(&runtime_id, bundle) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) - }) - .await + let bundle = serde_json::from_value::(request.bundle) + .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; + let result = api + .runtime + .sync_config_bundle(&runtime_id, bundle) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) } async fn check_runtime_config_bundle( @@ -31510,26 +29668,21 @@ async fn check_runtime_config_bundle( AxumPath((runtime_id, bundle_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let result = api - .runtime - .check_config_bundle( - &runtime_id, - ConfigBundleRef { - id: bundle_id, - digest: query.digest, - }, - ) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) - }) - .await + let result = api + .runtime + .check_config_bundle( + &runtime_id, + ConfigBundleRef { + id: bundle_id, + digest: query.digest, + }, + ) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) } #[cfg(test)] @@ -31549,7 +29702,7 @@ async fn dispatch_runtime_worker_input( worker_id: String, request: WorkerInputRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; let result = worker .input(&WorkerOperationContext::Backend, request) .await?; @@ -31560,16 +29713,11 @@ async fn delete_runtime_worker_uploaded_file( State(api): State, AxumPath((runtime_id, worker_id, artifact_id)): AxumPath<(String, String, String)>, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &artifact_id) - .map_err(|err| err.into_error())?; - Ok(Json(WorkerFileDeleteResponse { deleted: true })) - }) - .await + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &artifact_id) + .map_err(|err| err.into_error())?; + Ok(Json(WorkerFileDeleteResponse { deleted: true })) } async fn runtime_worker_completions( @@ -31577,18 +29725,13 @@ async fn runtime_worker_completions( AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, Json(request): Json, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let request = worker_completions_request_from_api(request)?; - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let result = api - .runtime - .worker_completions(&worker, request) - .map_err(|err| err.into_error())?; - Ok(Json(worker_completions_result_to_api(result)?)) - }) - .await + let request = worker_completions_request_from_api(request)?; + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let result = api + .runtime + .worker_completions(&worker, request) + .map_err(|err| err.into_error())?; + Ok(Json(worker_completions_result_to_api(result)?)) } fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource { @@ -31649,7 +29792,7 @@ async fn worker_protocol_ws( headers: HeaderMap, ws: WebSocketUpgrade, ) -> Response { - let worker = match WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await { + let worker = match WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) { Ok(worker) => worker, Err(error) => return ApiError::from(error).into_response(), }; @@ -32157,28 +30300,23 @@ async fn list_host_workers( State(api): State, AxumPath(host_id): AxumPath, ) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - let limit = api.config.max_records.min(200); - let runtime_workers = api - .runtime - .list_workers_for_host(&host_id, limit) - .map_err(|err| err.into_error())?; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::HostWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) - }) - .await + let limit = api.config.max_records.min(200); + let runtime_workers = api + .runtime + .list_workers_for_host(&host_id, limit) + .map_err(|err| err.into_error())?; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::HostWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) } fn project_workspace_worker_with_attachments( @@ -34737,27 +32875,9 @@ impl From for ApiError { } } -impl From for server_api::RepositoryApiError { - fn from(error: Error) -> Self { - ApiError::from(error).into_repository_api_error() - } -} - impl From for ApiError { fn from(error: Error) -> Self { let diagnostics = match &error { - Error::DbCapacityExceeded - | Error::DbExecutorUnavailable - | Error::DbOperationOutcomeUnknown => vec![RuntimeDiagnostic { - code: match &error { - Error::DbCapacityExceeded => "db_capacity_exceeded", - Error::DbExecutorUnavailable => "db_executor_unavailable", - _ => "db_operation_outcome_unknown", - } - .into(), - severity: HostDiagnosticSeverity::Error, - message: error.to_string(), - }], Error::WorkdirSessionOpenFailed { transport, .. } => vec![RuntimeDiagnostic { code: "workdir_session_open_failed".to_string(), severity: HostDiagnosticSeverity::Error, @@ -34981,8 +33101,6 @@ fn api_error_status(error: &Error) -> StatusCode { { StatusCode::SERVICE_UNAVAILABLE } - Error::DbCapacityExceeded | Error::DbExecutorUnavailable => StatusCode::SERVICE_UNAVAILABLE, - Error::DbOperationOutcomeUnknown => StatusCode::SERVICE_UNAVAILABLE, Error::WorkspaceSigningIdentity { .. } => StatusCode::SERVICE_UNAVAILABLE, Error::WorkdirSessionOpenFailed { .. } | Error::RuntimeOperationFailed { .. } => { StatusCode::BAD_GATEWAY @@ -35040,14 +33158,6 @@ impl IntoResponse for ApiError { } } -#[cfg(test)] -#[path = "server_db_execution_tests.rs"] -mod db_execution_tests; - -#[cfg(test)] -#[path = "server_assignment_db_execution_tests.rs"] -mod assignment_db_execution_tests; - #[cfg(test)] mod tests { mod assignment_delivery_tests { @@ -35991,73 +34101,6 @@ mod tests { ); } - #[tokio::test] - async fn cancelled_workdir_db_waiter_retains_session_lock_until_mutation_finishes() { - let dir = tempfile::tempdir().unwrap(); - init_clean_git_workspace(dir.path()); - let api = test_api(dir.path()).await; - let worker = RuntimeWorkerRef::new("guard-runtime", "guard-worker"); - let lock = current_worker_session_lock(&api, &worker); - let guard = Arc::new(lock.clone().lock_owned().await); - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let record = WorkdirRegistryRecord { - workspace_id: TEST_WORKSPACE_ID.to_string(), - workdir_id: "cancelled-waiter-workdir".to_string(), - display_name: Some("Accepted mutation".to_string()), - source: WorkdirRegistrySource::Repository { - runtime_id: worker.runtime_id.clone(), - repository_id: api.config.repositories[0].id.clone(), - }, - creation_selector: None, - creation_ref: None, - creation_tree: None, - current_selector: None, - current_ref: None, - current_tree: None, - observed_at_epoch_seconds: None, - materialization_status: "present".to_string(), - cleanliness: "unknown".to_string(), - created_at: now.clone(), - updated_at: now, - }; - let (started, accepted) = tokio::sync::oneshot::channel(); - let (release, released) = std::sync::mpsc::channel(); - let stage_api = api.clone(); - let waiter = tokio::spawn(async move { - run_worker_workdir_db_stage_locked(&stage_api, &guard, move |api| { - let _ = started.send(()); - released - .recv_timeout(std::time::Duration::from_secs(5)) - .map_err(|error| Error::Store(error.to_string()))?; - api.store.upsert_workdir_registry(&record)?; - Ok::<_, Error>(()) - }) - .await - }); - tokio::time::timeout(std::time::Duration::from_secs(5), accepted) - .await - .unwrap() - .unwrap(); - waiter.abort(); - assert!(waiter.await.unwrap_err().is_cancelled()); - // This is the same Worker lock, not an executor-capacity proxy. - let still_locked = lock.try_lock().is_err(); - release.send(()).unwrap(); - assert!( - still_locked, - "accepted mutation lost its Worker session guard" - ); - let _guard = tokio::time::timeout(std::time::Duration::from_secs(5), lock.lock()) - .await - .expect("accepted mutation did not release its Worker lock"); - let persisted = api - .store - .get_workdir_registry(TEST_WORKSPACE_ID, "cancelled-waiter-workdir") - .unwrap() - .expect("accepted mutation was cancelled with its waiter"); - assert_eq!(persisted.display_name.as_deref(), Some("Accepted mutation")); - } - #[tokio::test] async fn backend_expiry_fences_offline_grant_and_persists_cleanup_completion() { let dir = tempfile::tempdir().unwrap(); @@ -36251,10 +34294,10 @@ mod tests { inject_external_workdir_cleanup_failure(&grant.workdir_id); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = Arc::new(session_lock.lock_owned().await); + let session_guard = session_lock.lock().await; let error = tokio::time::timeout( std::time::Duration::from_secs(1), - open_current_worker_workdir_session_locked(&api, &worker, &link, &session_guard), + open_current_worker_workdir_session_locked(&api, &worker, &link), ) .await .expect("session-open expiry cleanup must not reacquire its Worker lock") @@ -36799,11 +34842,10 @@ mod tests { ); let session_lock = current_worker_session_lock(&api, &worker_ref); - let session_guard = Arc::new(session_lock.lock_owned().await); - let session = - open_current_worker_workdir_session_locked(&api, &worker_ref, link, &session_guard) - .await - .unwrap(); + let session_guard = session_lock.lock().await; + let session = open_current_worker_workdir_session_locked(&api, &worker_ref, link) + .await + .unwrap(); drop(session_guard); assert_eq!(session.capabilities(), expected_capabilities); if index == 0 { @@ -37226,15 +35268,11 @@ mod tests { workdir::WorkdirSessionCapabilities::ALL ); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = Arc::new(session_lock.lock_owned().await); - let broker_session = open_current_worker_workdir_session_locked( - &api, - &worker, - &external_link, - &session_guard, - ) - .await - .unwrap(); + let session_guard = session_lock.lock().await; + let broker_session = + open_current_worker_workdir_session_locked(&api, &worker, &external_link) + .await + .unwrap(); assert_eq!( broker_session.capabilities(), workdir::WorkdirSessionCapabilities::ALL @@ -38649,7 +36687,7 @@ mod tests { cached_status: "connected".to_owned(), timeout: std::time::Duration::from_secs(1), }); - SqliteWorkspaceStore::open_for_test(&api.config.database_path) + SqliteWorkspaceStore::open(&api.config.database_path) .unwrap() .upsert_workspace_runtime_binding( WorkspaceRuntimeBinding { @@ -40156,9 +38194,9 @@ mod tests { async fn workspace_api_does_not_register_a_missing_workspace() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); - let error = match WorkspaceApi::new_for_test(config, store.clone()).await { + let error = match WorkspaceApi::new(config, store.clone()).await { Ok(_) => panic!("missing Workspace registration must fail closed"), Err(error) => error, }; @@ -40175,12 +38213,11 @@ mod tests { let workspace = tempfile::tempdir().unwrap(); init_clean_git_workspace(workspace.path()); let config = test_server_config(workspace.path()); - let store = - Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(config.database_path.clone()).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); - let api = WorkspaceApi::new_for_test(config, store).await.unwrap(); + let api = WorkspaceApi::new(config, store).await.unwrap(); let workspace_id = api.config.workspace_id.clone(); let result = scoped_start_workspace_orchestrator( @@ -40235,7 +38272,7 @@ mod tests { let workspace = tempfile::tempdir().unwrap(); init_clean_git_workspace(workspace.path()); let config = test_server_config(workspace.path()); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -44779,8 +42816,8 @@ mod tests { let restored_config = test_server_config(&root) .with_embedded_runtime_store_root(root.join("restored-embedded-runtime")); let restored_store = - SqliteWorkspaceStore::open_for_test(restored_config.database_path.clone()).unwrap(); - let restored = WorkspaceApi::new_for_test(restored_config, Arc::new(restored_store)) + SqliteWorkspaceStore::open(restored_config.database_path.clone()).unwrap(); + let restored = WorkspaceApi::new(restored_config, Arc::new(restored_store)) .await .unwrap(); assert!( @@ -45339,7 +43376,7 @@ mod tests { } } - pub(super) struct DeterministicExecutionBackend { + struct DeterministicExecutionBackend { contexts: std::sync::Mutex< std::collections::HashMap< worker_runtime::identity::WorkerRef, @@ -45408,7 +43445,7 @@ mod tests { .store(true, std::sync::atomic::Ordering::SeqCst); } - pub(super) fn before_input_returns( + fn before_input_returns( &self, hook: impl FnOnce(&worker_runtime::identity::WorkerRef) + Send + 'static, ) { @@ -45448,7 +43485,7 @@ mod tests { execution.publish_event(event).unwrap(); } - pub(super) fn take_input_request_ids(&self) -> Vec> { + fn take_input_request_ids(&self) -> Vec> { std::mem::take( &mut *self .input_request_ids @@ -45457,7 +43494,7 @@ mod tests { ) } - pub(super) fn take_inputs(&self) -> Vec<(worker_runtime::identity::WorkerRef, String)> { + fn take_inputs(&self) -> Vec<(worker_runtime::identity::WorkerRef, String)> { std::mem::take(&mut *self.inputs.lock().expect("inputs lock")) } @@ -46714,17 +44751,6 @@ mod tests { .iter() .all(|(_, content)| { !content.contains("[Ticket item checker advisory]") }) ); - // Terminal SQL state is not actual cleanup completion. Keep this - // fixture's Tokio driver alive through accepted Host/cleanup work before - // dropping it, so a pending teardown timer cannot run on a dead runtime. - tokio::time::timeout(std::time::Duration::from_secs(5), async { - api.host_executors.host().unwrap().drain().await.unwrap(); - api.host_executors.cleanup().unwrap().drain().await.unwrap(); - api.worker_projection.shutdown_async().await; - api.store.db_executor().drain().await.unwrap(); - }) - .await - .expect("checker fixture must finish accepted cleanup before runtime shutdown"); } #[tokio::test] @@ -47117,8 +45143,7 @@ mod tests { notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), }; - let store = - Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(config.database_path.clone()).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); @@ -47256,8 +45281,7 @@ mod tests { drop(store); drop(execution); - let restarted_store = - SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let restarted_store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); let restarted_execution = Arc::new(DeterministicExecutionBackend::default()); restarted_execution.accept_restores(); let restarted = WorkspaceApi::new_with_execution_backend( @@ -47406,10 +45430,7 @@ mod tests { protocol::WorkerRunState::Running, )) })); - // Cleanup admission precedes its durable claim. The accepted job may - // still be queued, but neither queued nor executing ownership may stop - // this Run before the exact tracked finish marker below is published. - assert!(matches!( + assert_eq!( api.store .get_backend_job_attempt( TEST_WORKSPACE_ID, @@ -47419,8 +45440,8 @@ mod tests { .unwrap() .unwrap() .worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Executing) - )); + Some(BackendJobWorkerCleanupState::Executing) + ); execution.publish_worker_state( &worker, @@ -47447,7 +45468,7 @@ mod tests { async fn backend_job_delivery_recovery_replays_same_tracked_notification_id() { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend( config.clone(), @@ -48022,7 +46043,7 @@ mod tests { if let Some(parent) = config.database_path.parent() { fs::create_dir_all(parent).unwrap(); } - SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap() + SqliteWorkspaceStore::open(&config.database_path).unwrap() } #[tokio::test] @@ -48083,7 +46104,7 @@ mod tests { .unwrap(); let mut template = test_server_config(dir.path()); template.static_assets_dir = Some(static_dir); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); let app = build_workspace_server_router(template, store) .await .unwrap(); @@ -48166,7 +46187,7 @@ mod tests { async fn repository_free_workspace_can_be_created_selected_and_used_then_register_repository() { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); let owner = seed_test_api_token(store.as_ref(), "empty-owner"); let foreign = seed_test_api_token(store.as_ref(), "empty-foreign"); let app = build_workspace_server_router(config.clone(), store.clone()) @@ -48418,7 +46439,7 @@ mod tests { .. } = &config.auth; let expected_origin = expected_origin.clone(); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); store .upsert_account(&AccountRecord { account_id: "account-auth".to_owned(), @@ -49695,7 +47716,7 @@ mod tests { .unwrap(); let mut template = test_server_config(dir.path()); template.static_assets_dir = Some(static_dir); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); let token = seed_test_api_token(store.as_ref(), "two-workspaces"); let catalog = WorkspaceCatalogService::new( store.clone(), @@ -49836,7 +47857,7 @@ mod tests { let repository = dir.path().join("repository"); std::fs::create_dir_all(repository.join(".git")).unwrap(); let template = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&template.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&template.database_path).unwrap()); let app = build_workspace_server_router(template, store) .await .unwrap(); @@ -49905,7 +47926,7 @@ mod tests { async fn memory_consolidation_backlog_ignores_legacy_filesystem_staging() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -49938,7 +47959,7 @@ mod tests { async fn memory_consolidation_reuses_existing_idle_worker() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -53379,7 +51400,7 @@ mod tests { workspace_root: impl Into, ) -> (WorkspaceApi, Arc) { let config = test_server_config(workspace_root); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) @@ -53403,7 +51424,7 @@ mod tests { config.repositories[0].source = source; config.repositories[0].path = None; config.repositories[0].default_selector = Some("develop".to_string()); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -53432,7 +51453,7 @@ mod tests { docs.path = Some(docs_root); docs.default_selector = Some("develop".to_string()); config.repositories.push(docs); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -53867,7 +51888,6 @@ mod tests { store, Arc::new(DeterministicExecutionBackend::default()), BackendResourceBroker::default(), - HostExecutors::default(), Some(dispatcher.clone()), ) .await @@ -54113,7 +52133,7 @@ mod tests { cached_status: "connected".to_string(), timeout: std::time::Duration::from_secs(1), }); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); let owner_account_id = format!("account-{TEST_WORKSPACE_ID}"); store .upsert_account(&crate::store::AccountRecord { @@ -54413,8 +52433,7 @@ mod tests { let mut revoked = trust; revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string()); - let authority = - SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(); + let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap(); authority .upsert_workspace_runtime_binding(revoked, true) .unwrap(); @@ -55344,7 +53363,7 @@ mod tests { .unwrap() .digest ); - let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); assert_eq!( changed.digest, restarted @@ -55890,7 +53909,7 @@ mod tests { .unwrap(); assert_eq!(capability_changed.items, reattached.items); assert_ne!(capability_changed.digest, reattached.digest); - let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); let (restarted_page, restarted_digest) = restarted .list_worker_workdir_links_page_with_digest( TEST_WORKSPACE_ID, @@ -55909,7 +53928,7 @@ mod tests { .unwrap() .run_to_completion(128, std::time::Duration::from_millis(1), None) .unwrap(); - let restored = SqliteWorkspaceStore::from_connection_for_test(snapshot).unwrap(); + let restored = SqliteWorkspaceStore::from_connection(snapshot).unwrap(); assert_eq!( restored .list_worker_workdir_links_page_with_digest( @@ -55954,7 +53973,7 @@ mod tests { ) .unwrap(); assert_ne!(read_write_digest, capability_changed.digest); - let reader = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); + let reader = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); let barrier = Arc::new(std::sync::Barrier::new(2)); std::thread::scope(|scope| { let worker = &fixture.worker; @@ -57118,8 +55137,6 @@ mod tests { .unwrap(); let worker = orchestrator.worker.unwrap(); let worker = RuntimeWorkerRef::new(worker.runtime_id, worker.worker_id); - let mut stop_events = api.worker_projection.subscribe_ordered(10).unwrap(); - let _ = stop_events.take_snapshot(); api.runtime .stop_worker( &worker, @@ -57129,25 +55146,6 @@ mod tests { }, ) .unwrap(); - // A synchronous Runtime stop no longer implies that its queued SQL - // projection was published. Observe stopped before taking the removal - // baseline, rather than mistaking its Upsert for the atomic removal. - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - let event = stop_events.recv().await.unwrap(); - if event.changes.iter().any(|change| matches!( - change, - crate::store::WorkerCatalogChange::Upsert(record) - if record.registry.worker == worker - && record.observation.as_ref().is_some_and(|observation| - observation.worker.state == protocol::subscription::SubscriptionWorkerState::Stopped) - )) { - break; - } - } - }) - .await - .expect("stopped Runtime observation must commit before removal baseline"); let worker_root = workspace .path() .join(".test-embedded-runtime-store/workers") @@ -57809,7 +55807,7 @@ mod tests { ) .unwrap(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.clone(), + api.store.as_ref(), &api.config, &proof, other_workspace, @@ -57835,7 +55833,7 @@ mod tests { let identity = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-test").unwrap(); configure_runtime_request_auth(&mut api, &identity, "runtime-test"); - let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); let memory_settings = store .get_workspace_memory_settings(TEST_WORKSPACE_ID) .unwrap(); @@ -57870,7 +55868,7 @@ mod tests { let proof = issue(); let source = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.clone(), + api.store.as_ref(), &api.config, &proof, TEST_WORKSPACE_ID, @@ -57888,7 +55886,7 @@ mod tests { .unwrap(); let proof = issue(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.clone(), + api.store.as_ref(), &api.config, &proof, TEST_WORKSPACE_ID, @@ -60035,9 +58033,9 @@ mod tests { std::fs::write(static_dir.join("index.html"), "
Yoi Workspace
").unwrap(); std::fs::write(static_dir.join("assets/app.js"), "console.log('yoi');").unwrap(); - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); let mut config = test_server_config(dir.path()); - let sqlite_store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); + let sqlite_store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); sqlite_store .upsert_workspace(&WorkspaceRecord { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -60072,7 +58070,7 @@ mod tests { "API Ticket", ticket::TicketState::Open, ); - let sqlite_store = SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap(); + let sqlite_store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); sqlite_store .upsert_workspace(&WorkspaceRecord { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -62596,7 +60594,7 @@ mod tests { async fn objective_mutation_endpoints_round_trip_through_workspace_authority() { let dir = tempfile::tempdir().unwrap(); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); let owner_account_id = format!("account-{TEST_WORKSPACE_ID}"); store .upsert_account(&crate::store::AccountRecord { diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs index 33ef01ed4..0fcb072af 100644 --- a/crates/workspace-server/src/server/subjektiv_job.rs +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -65,85 +65,70 @@ pub(super) fn require_candidate_grant( /// candidate decision. Once Idle is confirmed, stop cancels any legacy surface /// background work; the new Job regenerates it and does not infer surface success. pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult { - let job_api = api.clone(); - let subject_id = subject_id.to_owned(); let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); - let (worker, complete) = api - .host_executors - .host()? - .run(move || { - let api = &job_api; - let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); - let Some(owner) = api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - else { - return Ok((None, true)); - }; - let mut summary = api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())?; - summary.singleton_key = Some(owner.key.clone()); - require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; - let stopped = summary.state.eq_ignore_ascii_case("stopped"); - let idle = summary.state.eq_ignore_ascii_case("idle") - && summary - .worker_state - .as_ref() - .is_some_and(|s| s.state == protocol::WorkerState::Idle); - if !stopped && !idle { - return Ok((None, false)); - } - if api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - .as_ref() - .map(|o| &o.worker) - != Some(&owner.worker) - { - return Err(Error::RepositoryConflict( - "legacy consolidation owner moved during cutover".into(), - ) - .into()); - } - if !summary.state.eq_ignore_ascii_case("stopped") { - let stopped = api - .runtime - .stop_worker( - &owner.worker, - WorkerLifecycleRequest { - reason: Some( - "Retire legacy subjektiv consolidator after foreground completion" - .into(), - ), - ticket_assignment: None, - }, - ) - .map_err(|e| e.into_error())?; - if stopped.state != InternalWorkerOperationState::Accepted { - return Ok((None, false)); - } - } - if !api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())? - .state - .eq_ignore_ascii_case("stopped") - { - return Ok((None, false)); - } - Ok::<_, ApiError>((Some(owner.worker), false)) - }) - .await?; - let Some(worker) = worker else { - return Ok(complete); + let Some(owner) = api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + else { + return Ok(true); }; + let mut summary = api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())?; + summary.singleton_key = Some(owner.key.clone()); + require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; + let stopped = summary.state.eq_ignore_ascii_case("stopped"); + let idle = summary.state.eq_ignore_ascii_case("idle") + && summary + .worker_state + .as_ref() + .is_some_and(|s| s.state == protocol::WorkerState::Idle); + if !stopped && !idle { + return Ok(false); + } + if api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .as_ref() + .map(|o| &o.worker) + != Some(&owner.worker) + { + return Err(Error::RepositoryConflict( + "legacy consolidation owner moved during cutover".into(), + ) + .into()); + } + if !summary.state.eq_ignore_ascii_case("stopped") { + let stopped = api + .runtime + .stop_worker( + &owner.worker, + WorkerLifecycleRequest { + reason: Some( + "Retire legacy subjektiv consolidator after foreground completion".into(), + ), + ticket_assignment: None, + }, + ) + .map_err(|e| e.into_error())?; + if stopped.state != InternalWorkerOperationState::Accepted { + return Ok(false); + } + } + if !api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())? + .state + .eq_ignore_ascii_case("stopped") + { + return Ok(false); + } let removal = WorkerRemovalService::new(api) .execute_target_removal( api.runtime.as_ref(), - &worker, + &owner.worker, "subjektiv consolidation Job cutover", None, ) @@ -156,17 +141,10 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) - )) .into()); } - let store = api.store.clone(); - let workspace_id = api.workspace_id().to_owned(); - api.store - .db_executor() - .run(move || { - store - .current_worker_singleton_owner(&workspace_id, &key) - .map(|owner| owner.is_none()) - .map_err(ApiError::from) - }) - .await + Ok(api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .is_none()) } fn bounded_candidate_batch( diff --git a/crates/workspace-server/src/server/tests/api_logging_tests.rs b/crates/workspace-server/src/server/tests/api_logging_tests.rs index 2bac1950b..e4095b829 100644 --- a/crates/workspace-server/src/server/tests/api_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/api_logging_tests.rs @@ -806,19 +806,7 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc .unwrap(); }; let ((status, body, events), ()) = tokio::join!( - async { - let result = request_with_logs(build_router(fixture.api.clone()), request).await; - // A pre-dispatch rejection leaves the fake provider waiting on an - // open channel. Fail with the actual response here instead of - // hanging join! until the outer suite timeout hides that cause. - assert_eq!( - result.0, - StatusCode::BAD_GATEWAY, - "early command-start response: {}", - result.1 - ); - result - }, + request_with_logs(build_router(fixture.api.clone()), request), provider ); assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); @@ -844,78 +832,3 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc "post-dispatch failure was retried" ); } - -#[tokio::test] -async fn saturated_host_before_command_start_returns_availability_without_a_provider_operation() { - let mut fixture = manual_worker_assignment_fixture().await; - let identity = - worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) - .unwrap(); - configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); - let (sender, mut commands) = tokio::sync::mpsc::channel(1); - let connection = Arc::new(ExternalProviderConnection { - grant_id: "overload-grant".into(), - workdir_id: fixture.main_workdir_id.clone(), - provider_instance_id: "overload-provider".into(), - generation: 1, - expires_at: None, - capabilities: workdir::WorkdirSessionCapabilities::ALL, - admission: Arc::new(tokio::sync::Semaphore::new(1)), - shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), - sender, - }); - fixture.runtime.workdir_session_factory = Some(Arc::new(move |_| { - Ok(Arc::new(ExternalProviderWorkdirSession::new( - connection.clone(), - None, - )) as WorkdirSessionHandle) - })); - fixture - .api - .runtime - .register_or_replace(fixture.runtime.clone()); - let host = fixture.api.host_executors.host().unwrap(); - host.drain().await.unwrap(); - let mut releases = Vec::new(); // dropping senders also releases jobs on panic - for _ in 0..2 { - let (release, permit) = std::sync::mpsc::channel::<()>(); - releases.push(release); - let (started, accepted) = tokio::sync::oneshot::channel(); - host.dispatch(move || { - let _ = started.send(()); - let _ = permit.recv_timeout(std::time::Duration::from_secs(10)); - }) - .unwrap(); - accepted.await.unwrap(); - } - for _ in 0..16 { - host.dispatch(|| {}).unwrap(); - } - let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations"); - let request = runtime_source_request( - &identity, - Some(&fixture.worker.worker_id), - "POST", - &path, - serde_json::to_vec(&json!({ - "target_workdir": "checkout", - "operation": {"operation": "command_start", "request": { - "command": "not-dispatched", "timeout_secs": 1, "output_limit": 1024, - "cwd": "", "spill_dir": null, "tool_call_id": "not-dispatched" - }} - })) - .unwrap(), - ); - let (status, body, _) = request_with_logs(build_router(fixture.api.clone()), request).await; - assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); - assert!(body.to_string().contains("db_capacity_exceeded"), "{body}"); - // The response is already final but the live fake-provider receiver remains - // Pending. An unbounded join of those two futures hid this causal rejection - // behind the original 600-second suite timeout. No process or command ran. - let provider_command = commands.recv(); - tokio::pin!(provider_command); - assert!(futures::poll!(provider_command.as_mut()).is_pending()); - drop(releases); - host.drain().await.unwrap(); - fixture.api.store.db_executor().drain().await.unwrap(); -} diff --git a/crates/workspace-server/src/server/tests/auth_logging_tests.rs b/crates/workspace-server/src/server/tests/auth_logging_tests.rs index d5bf3334a..b4b51d7bc 100644 --- a/crates/workspace-server/src/server/tests/auth_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/auth_logging_tests.rs @@ -54,7 +54,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_ configure_runtime_request_auth(&mut api, &identity, "runtime-test"); let revoked = RuntimeIdentityMaterial::generate("revoked-runtime-secret").unwrap(); configure_runtime_request_auth(&mut api, &revoked, "revoked-runtime-secret"); - SqliteWorkspaceStore::open_for_test(&api.config.database_path) + SqliteWorkspaceStore::open(&api.config.database_path) .unwrap() .revoke_workspace_runtime_binding( TEST_WORKSPACE_ID, @@ -70,7 +70,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_ let body = "body-secret"; let apps = [ build_router(api.clone()), - workspace_server_router(WorkspaceServerApi::new_for_test( + workspace_server_router(WorkspaceServerApi::new( api.config.clone(), api.store.clone(), )), @@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m &error, ) }); - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); assert!( to_bytes(response.into_body(), usize::MAX) .await @@ -347,7 +347,7 @@ async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() { let path = format!("/api/w/{TEST_WORKSPACE_ID}/repositories"); for app in [ build_router(api.clone()), - workspace_server_router(WorkspaceServerApi::new_for_test( + workspace_server_router(WorkspaceServerApi::new( api.config.clone(), api.store.clone(), )), diff --git a/crates/workspace-server/src/server/tests/drive_tests.rs b/crates/workspace-server/src/server/tests/drive_tests.rs index 3b7ec65ca..1d05ff7db 100644 --- a/crates/workspace-server/src/server/tests/drive_tests.rs +++ b/crates/workspace-server/src/server/tests/drive_tests.rs @@ -40,7 +40,7 @@ async fn cached_drive_handle_rejects_mutation_after_deletion_reservation() { }) .await .unwrap(); - let api = WorkspaceApi::new_for_test(config.clone(), store.clone()) + let api = WorkspaceApi::new(config.clone(), store.clone()) .await .unwrap(); let cached = api.drive().await.unwrap(); @@ -104,7 +104,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature // Use a separate embedded-runtime namespace for the first API: this test // reconstructs Drive authority in-process rather than spawning/restarting // product processes (runtime observers may retain their namespace). - let first_api = WorkspaceApi::new_for_test(config.clone(), store.clone()) + let first_api = WorkspaceApi::new(config.clone(), store.clone()) .await .unwrap(); // Merely constructing a Workspace API must not create blob storage. @@ -160,7 +160,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature .unwrap() .unwrap(); drop(first_api); - let recovered = WorkspaceServerApi::new_for_test(config.clone(), store.clone()); + let recovered = WorkspaceServerApi::new(config.clone(), store.clone()); let api = recovered .api_for_workspace(&config.workspace_id) .await diff --git a/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs b/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs index 71efdb137..02ab40e35 100644 --- a/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs +++ b/crates/workspace-server/src/server/tests/drive_web_roundtrip_tests.rs @@ -39,10 +39,10 @@ impl RealDriveFixture { let temp = tempfile::tempdir().unwrap(); let mut config = test_server_config(temp.path()); config.repositories.clear(); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); let owner = seed_test_api_token(store.as_ref(), "drive-web-owner"); let second = seed_test_api_token(store.as_ref(), "drive-web-second"); - let server_api = WorkspaceServerApi::new_for_test(config, store.clone()); + let server_api = WorkspaceServerApi::new(config, store.clone()); let server = LoopbackServer::start(workspace_server_router(server_api.clone())).await; // Exercise T-717's real repositoryless Workspace bootstrap, not a // pre-existing test Workspace with its repository merely hidden in UI. diff --git a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs index 358b9a7bb..7a4066b39 100644 --- a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs +++ b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs @@ -767,9 +767,6 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) { std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 }) ); assert!(execution.protocol_methods().is_empty()); - // SQLite jobs run on external threads; paused Tokio time cannot observe their - // progress and may auto-advance a reconnect timeout before SQL completes. - tokio::time::resume(); assert_same_worker_reconnect_and_stop(&worker, &context).await; } @@ -847,7 +844,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress() { let workspace = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs index 37519b22c..b54c4bdad 100644 --- a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs +++ b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs @@ -27,7 +27,7 @@ async fn subject_session_callback_completes_while_create_and_deletion_are_waitin ) .unwrap(); let worker_id = reserved.worker_id.to_string(); - let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); + let server = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); // Reuse the fixture API, but exercise the real outer dispatcher and both auth layers. server .apis diff --git a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs index 733df3809..64ca15dc3 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs @@ -144,7 +144,7 @@ impl LaunchFixture { async fn new(remote: bool) -> Self { let temp = tempfile::tempdir().unwrap(); let config = test_server_config(temp.path()); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); let backend = Arc::new(LaunchExecutionBackend::default()); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), backend.clone()) diff --git a/crates/workspace-server/src/server/tests/worker_drive_tests.rs b/crates/workspace-server/src/server/tests/worker_drive_tests.rs index 481c309ad..2b24e45a5 100644 --- a/crates/workspace-server/src/server/tests/worker_drive_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_drive_tests.rs @@ -151,7 +151,7 @@ impl Fixture { let temp = tempfile::tempdir().unwrap(); let mut config = test_server_config(temp.path()); config.repositories.clear(); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); let mut api = WorkspaceApi::new_with_execution_backend( config, store, diff --git a/crates/workspace-server/src/server/tests/worker_operations_tests.rs b/crates/workspace-server/src/server/tests/worker_operations_tests.rs index 574c48b8e..2229bdb59 100644 --- a/crates/workspace-server/src/server/tests/worker_operations_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_operations_tests.rs @@ -830,11 +830,7 @@ async fn grant_replacement_while_admission_waits_cannot_use_another_grants_lock( }; let admission = handle.set_pinned(&context, true); tokio::pin!(admission); - // The first poll can now be waiting for the offloaded authority read, not - // the held grant lock. Drain this fixture's actual SQL, then poll its saved - // reply to reach admission before replacing authority. - assert!(futures::poll!(admission.as_mut()).is_pending()); - api.store.db_executor().drain().await.unwrap(); + // Poll deterministically to the held lock before replacing grant authority. assert!(futures::poll!(admission.as_mut()).is_pending()); api.store .revoke_worker_control_grant( @@ -1333,9 +1329,8 @@ async fn restore_guard_internal_stale_conflict_retires_intent_for_later_explicit ); assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); // Retirement, unlike in-memory refresh, must survive caller journal restart. - api.config_store = Arc::new( - SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(), - ); + api.config_store = + Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); let fresh = api.runtime.worker(&identity).unwrap(); let next = api .config_store @@ -1495,7 +1490,7 @@ async fn restore_guard_internal_intent_survives_client_store_restart_without_rep ); // Reopen the durable caller journal, discarding all client-side memory of request identity. api.config_store = - Arc::new(SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap()); + Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); let pinned = api .config_store .pin_internal_worker_restore_intent( @@ -1604,7 +1599,7 @@ async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_w crate::repository_source::repository_source_fingerprint(&source); config.repositories[0].source = source; config.repositories[0].path = None; - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); let execution = Arc::new(DeterministicExecutionBackend::default()); execution.accept_restores(); let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs index 142ae143e..9bfbd3364 100644 --- a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs @@ -56,7 +56,7 @@ impl Fixture { api.runtime.register_or_replace(runtime); let now = Arc::new(AtomicI64::new(START)); let clock = now.clone(); - let mut server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); + let mut server = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); server.proof_clock = Arc::new(move || clock.load(Ordering::SeqCst)); server .apis @@ -330,38 +330,9 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin let f = Fixture::new().await; let mut a = f.attach("1"); let (first, reply) = f.start_read("1", &mut a).await; - let request = f.operation("1"); - let proof = request - .headers() - .get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER) - .unwrap() - .to_str() - .unwrap(); - let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof) - .unwrap() - .jti; - let mut second = Box::pin(f.app.clone().oneshot(request)); - // DB auth is now genuinely asynchronous. Observe durable proof consumption, - // not a guessed number of polls, before advancing the injected proof clock. - let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap(); - tokio::time::timeout(BOUND, async { - loop { - assert!(futures::poll!(&mut second).is_pending()); - let consumed: bool = connection - .query_row( - "SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)", - [&jti], - |row| row.get(0), - ) - .unwrap(); - if consumed { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("second request never authenticated before resource wait"); + let mut second = Box::pin(f.app.clone().oneshot(f.operation("1"))); + // Poll through dispatch/auth until the handler waits on its resource lock. + assert!(futures::poll!(&mut second).is_pending()); assert!( a.try_recv().is_err(), "same session read escaped resource exclusion" @@ -371,12 +342,7 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin assert_json(first.await.unwrap(), StatusCode::OK).await; assert!(futures::poll!(&mut second).is_pending()); let Some(ExternalProviderCommand::Operation { response, .. }) = - tokio::time::timeout(BOUND, async { - tokio::select! { - command = a.recv() => command, - result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()), - } - }).await.unwrap() + tokio::time::timeout(BOUND, a.recv()).await.unwrap() else { panic!("expected second read") }; @@ -882,18 +848,12 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits( }), ); let mut deletion = Box::pin(f.app.clone().oneshot(request)); + assert!(futures::poll!(&mut deletion).is_pending()); let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await; - tokio::time::timeout(BOUND, async { - loop { - assert!(futures::poll!(&mut deletion).is_pending()); - if gate.admit(false).is_none() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("deletion start did not fence admission after async authentication"); + assert!( + gate.admit(false).is_none(), + "deletion start did not fence admission" + ); let session_id = session_store::new_session_id().to_string(); let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions"); let callback = f.request( diff --git a/crates/workspace-server/src/server/worker_operations.rs b/crates/workspace-server/src/server/worker_operations.rs index 447ae7e29..35b60148f 100644 --- a/crates/workspace-server/src/server/worker_operations.rs +++ b/crates/workspace-server/src/server/worker_operations.rs @@ -57,19 +57,6 @@ impl WorkerOperationContext { } Ok(()) } - pub(super) async fn from_request_async( - api: &WorkspaceApi, - context: &server_api::ServerRequestContext, - ) -> std::result::Result { - let owned_api = api.clone(); - let context = context.clone(); - api.store - .db_executor() - .run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context))) - .await - .map_err(|error| ApiError::from(error).into_repository_api_error())? - } - pub(super) fn from_request( api: &WorkspaceApi, context: &server_api::ServerRequestContext, @@ -175,20 +162,8 @@ impl WorkspaceWorker { source: crate::worker_source::VerifiedWorkerMutationSource, reason: &str, ) -> std::result::Result { - let removal = self.removal.clone(); - let identity = self.identity.clone(); - self.removal - .store - .db_executor() - .run(move || { - removal - .store - .get_worker_registry(&removal.workspace_id, &identity)? - .ok_or_else(|| Error::UnknownWorker { worker: identity }) - .map(drop) - }) - .await - .map_err(|error| error.to_string())?; + self.current_record() + .map_err(|error| error.error.to_string())?; self.removal .execute_async( source, @@ -250,20 +225,6 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor } impl WorkspaceWorker { - pub(crate) async fn resolve_async( - api: &WorkspaceApi, - runtime_id: &str, - reference: &str, - ) -> Result { - let owned_api = api.clone(); - let runtime_id = runtime_id.to_owned(); - let reference = reference.to_owned(); - api.store - .db_executor() - .run(move || Self::resolve(&owned_api, &runtime_id, &reference)) - .await - } - pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result { Self::resolve_with_services( api.clone(), @@ -278,36 +239,7 @@ impl WorkspaceWorker { context: &WorkerOperationContext, permission: &str, ) -> ApiResult>> { - let worker = self.clone(); - let initial_context = context.clone(); - let permission = permission.to_owned(); - let initial_permission = permission.clone(); - let grant = self - .api - .store - .db_executor() - .run(move || -> ApiResult> { - worker.current_record()?; - let WorkerOperationContext::WorkerControl { controller } = initial_context else { - return Ok(None); - }; - if worker - .api - .store - .get_worker_registry(worker.api.workspace_id(), &controller)? - .is_none() - { - return Err(Error::UnknownWorker { worker: controller }.into()); - } - Ok(Some(authorize_known_worker_permission( - &worker.api, - worker.api.workspace_id(), - &controller, - &worker.identity, - &initial_permission, - )?)) - }) - .await?; + self.current_record()?; match context { #[cfg(test)] WorkerOperationContext::Backend => Ok(None), @@ -316,35 +248,42 @@ impl WorkspaceWorker { Ok(None) } WorkerOperationContext::WorkerControl { controller } => { - let grant = grant.expect("Worker control context resolved a grant"); - // Await admission outside the DB lane; the recheck owns the guard - // until it finishes, even if its async waiter is cancelled. + if self + .api + .store + .get_worker_registry(self.api.workspace_id(), controller)? + .is_none() + { + return Err(Error::UnknownWorker { + worker: controller.clone(), + } + .into()); + } + let grant = authorize_known_worker_permission( + &self.api, + self.api.workspace_id(), + controller, + &self.identity, + permission, + )?; let guard = worker_control_lock(&self.api, &grant.grant_id) .lock_owned() .await; - let worker = self.clone(); - let controller = controller.clone(); - self.api - .store - .db_executor() - .run(move || -> ApiResult<_> { - worker.current_record()?; - let current_grant = authorize_known_worker_permission( - &worker.api, - worker.api.workspace_id(), - &controller, - &worker.identity, - &permission, - )?; - if current_grant.grant_id != grant.grant_id { - return Err(Error::RepositoryConflict( - "Worker control grant changed while waiting for admission".into(), - ) - .into()); - } - Ok(Some(guard)) - }) - .await + self.current_record()?; + let current_grant = authorize_known_worker_permission( + &self.api, + self.api.workspace_id(), + controller, + &self.identity, + permission, + )?; + if current_grant.grant_id != grant.grant_id { + return Err(Error::RepositoryConflict( + "Worker control grant changed while waiting for admission".into(), + ) + .into()); + } + Ok(Some(guard)) } } } @@ -354,16 +293,8 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult { - let guard = self.authorize_operation(context, "restore").await?; - let worker = self.clone(); - self.api - .host_executors - .host()? - .run(move || { - let _guard = guard; - worker.restore_internal(request) - }) - .await + let _guard = self.authorize_operation(context, "restore").await?; + self.restore_internal(request) } /// Trusted Backend callers still supply an explicit observed intent. Runtime @@ -381,48 +312,27 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let guard = self.authorize_operation(context, "stop").await?; - let worker = self.clone(); - let (result, guard) = self + let _guard = self.authorize_operation(context, "stop").await?; + let result = self .api - .host_executors - .host()? - .run(move || -> ApiResult<_> { - let result = worker - .api - .runtime - .stop_worker(&worker.identity, request) - .map_err(|e| e.into_error())?; - Ok((result, guard)) - }) - .await?; + .runtime + .stop_worker(&self.identity, request) + .map_err(|e| e.into_error())?; if result.state != InternalWorkerOperationState::Accepted { return Ok(result); } parse_runtime_worker_id_for_registry(&self.identity.worker_id)?; let session_lock = current_worker_session_lock(&self.api, &self.identity); - let session_guard = session_lock.lock_owned().await; + let _session_guard = session_lock.lock().await; close_current_worker_session_locked(&self.api, &self.identity).await?; - let worker = self.clone(); - self.api - .host_executors - .host()? - .run(move || -> ApiResult<_> { - let (_control, _session) = (guard, session_guard); - if let Some(record) = worker - .api - .store - .get_worker_registry(worker.api.workspace_id(), &worker.identity)? - { - sync_linked_workdir_after_worker_stop( - &worker.api, - &worker.identity.runtime_id, - &record, - )?; - } - Ok(result) - }) - .await + if let Some(record) = self + .api + .store + .get_worker_registry(self.api.workspace_id(), &self.identity)? + { + sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?; + } + Ok(result) } pub(super) async fn cancel( @@ -430,20 +340,12 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let guard = self.authorize_operation(context, "cancel").await?; - let worker = self.clone(); - self.api - .host_executors - .host()? - .run(move || -> ApiResult<_> { - let _guard = guard; - Ok(worker - .api - .runtime - .cancel_worker(&worker.identity, request) - .map_err(|e| e.into_error())?) - }) - .await + let _guard = self.authorize_operation(context, "cancel").await?; + Ok(self + .api + .runtime + .cancel_worker(&self.identity, request) + .map_err(|e| e.into_error())?) } pub(super) async fn input( @@ -456,61 +358,46 @@ impl WorkspaceWorker { } else { "send_input" }; - let guard = self.authorize_operation(context, permission).await?; + let _guard = self.authorize_operation(context, permission).await?; let session_lock = current_worker_session_lock(&self.api, &self.identity); - let session_guard = session_lock.lock_owned().await; - self.ensure_interactive().await?; + let _session_guard = session_lock.lock().await; + self.ensure_interactive()?; if matches!(context, WorkerOperationContext::WorkerControl { .. }) && request.kind == WorkerInputKind::User { request.kind = WorkerInputKind::UserIfIdle; } - let worker = self.clone(); - self.api - .host_executors - .host()? - .run(move || -> ApiResult<_> { - let (_control, _session) = (guard, session_guard); - Ok(worker - .api - .runtime - .send_input(&worker.identity, request) - .map_err(|e| e.into_error())?) - }) - .await + Ok(self + .api + .runtime + .send_input(&self.identity, request) + .map_err(|e| e.into_error())?) } - async fn ensure_interactive(&self) -> ApiResult<()> { - let worker = self.clone(); - self.api + fn ensure_interactive(&self) -> ApiResult<()> { + if let Some(claim) = self + .api .store - .db_executor() - .run(move || -> ApiResult<()> { - if let Some(claim) = worker - .api - .store - .get_pending_ticket_assignment_operation_for_worker( - worker.api.workspace_id(), - &worker.identity, - )? - { - return Err(Error::TicketAssignmentConflict(format!( - "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", - claim.operation_id, - )).into()); - } - if worker - .api - .store - .worker_registry_projection(worker.api.workspace_id(), &worker.identity)? - .and_then(|projection| projection.job) - .is_some() - { - return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); - } - Ok(()) - }) - .await + .get_pending_ticket_assignment_operation_for_worker( + self.api.workspace_id(), + &self.identity, + )? + { + return Err(Error::TicketAssignmentConflict(format!( + "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", + claim.operation_id, + )).into()); + } + if self + .api + .store + .worker_registry_projection(self.api.workspace_id(), &self.identity)? + .and_then(|projection| projection.job) + .is_some() + { + return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); + } + Ok(()) } pub(super) async fn set_pinned( @@ -518,44 +405,33 @@ impl WorkspaceWorker { context: &WorkerOperationContext, pinned: bool, ) -> ApiResult { - let guard = self.authorize_operation(context, "pin").await?; - let worker = self.clone(); - self.api - .store - .db_executor() - .run(move || -> ApiResult { - // Publication and its DB mutation are one owned stage. Accepted - // work retains control admission through commit and publication. - let _guard = guard; - let retention_state = if pinned { "pinned" } else { "normal" }; - let changed = worker.api.worker_projection.publish_ordered(|| { - worker - .api - .store - .update_worker_retention( - worker.api.workspace_id(), - &worker.identity, - retention_state, - now_registry_timestamp().as_str(), - ) - .map(|commit| (!commit.changes.is_empty(), commit)) - })?; - if !changed { - return Err(cleanup_api_error( - &worker.identity.runtime_id, - "workspace_worker_retention_unknown_worker", - "Worker is not known to the Backend registry", - )); - } - Ok(WorkerRetentionResponse { - workspace_id: worker.api.workspace_id().to_owned(), - runtime_id: worker.identity.runtime_id.clone(), - worker_id: worker.identity.worker_id.clone(), - pinned, - retention_state: retention_state.into(), - }) - }) - .await + let _guard = self.authorize_operation(context, "pin").await?; + let retention_state = if pinned { "pinned" } else { "normal" }; + let changed = self.api.worker_projection.publish_ordered(|| { + self.api + .store + .update_worker_retention( + self.api.workspace_id(), + &self.identity, + retention_state, + now_registry_timestamp().as_str(), + ) + .map(|commit| (!commit.changes.is_empty(), commit)) + })?; + if !changed { + return Err(cleanup_api_error( + &self.identity.runtime_id, + "workspace_worker_retention_unknown_worker", + "Worker is not known to the Backend registry", + )); + } + Ok(WorkerRetentionResponse { + workspace_id: self.api.workspace_id().to_owned(), + runtime_id: self.identity.runtime_id.clone(), + worker_id: self.identity.worker_id.clone(), + pinned, + retention_state: retention_state.into(), + }) } /// The caller owns the full cleanup-plan digest check. The removal service @@ -735,7 +611,7 @@ impl WorkspaceWorkerMethodSender { .await?; let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity); let _session_guard = session_lock.lock().await; - self.worker.ensure_interactive().await?; + self.worker.ensure_interactive()?; let WorkerOperationContext::Browser { source, .. } = context else { return Err(Error::WorkspacePermissionDenied( "Browser Worker protocol context required".into(), diff --git a/crates/workspace-server/src/server/workspace_admission.rs b/crates/workspace-server/src/server/workspace_admission.rs index 5f8335629..06c94b9a8 100644 --- a/crates/workspace-server/src/server/workspace_admission.rs +++ b/crates/workspace-server/src/server/workspace_admission.rs @@ -72,28 +72,6 @@ pub(super) struct RequestLease { callback: bool, } -impl crate::db_executor::DbOperationLifetime for RequestLease { - fn retain(&self) -> Box { - // This is continuation of an already admitted request, not fresh admission. - // Retain its drain responsibility before transferring work to the DB owner; - // dropping/aborting the HTTP waiter must not allow deletion past live SQL. - let mut state = self - .gate - .state - .lock() - .expect("Workspace admission poisoned"); - if self.callback { - state.callbacks += 1; - } else { - state.requests += 1; - } - Box::new(RequestLease { - gate: self.gate.clone(), - callback: self.callback, - }) - } -} - impl Drop for RequestLease { fn drop(&mut self) { let mut state = self diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index ee20767bc..7f8c57d56 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -597,7 +597,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) { let dir = tempfile::tempdir().unwrap(); init_clean_git_workspace(dir.path()); let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); diff --git a/crates/workspace-server/src/server_assignment_db_execution_tests.rs b/crates/workspace-server/src/server_assignment_db_execution_tests.rs deleted file mode 100644 index 724066895..000000000 --- a/crates/workspace-server/src/server_assignment_db_execution_tests.rs +++ /dev/null @@ -1,1408 +0,0 @@ -//! Real SQLite and generated HTTP boundaries for continuing assignment input. -//! Every fixture owns its finite DB/Host lanes and all observer userdata. -use super::tests::DeterministicExecutionBackend; -use super::*; -use crate::SqliteWorkspaceStore; -use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime}; -use rusqlite::{Connection, ffi}; -use std::{ - ffi::c_void, - future::Future, - num::NonZeroUsize, - sync::{ - atomic::{AtomicBool, AtomicUsize}, - mpsc, - }, - thread::JoinHandle, - time::Duration as StdDuration, -}; -use tokio::sync::oneshot; -use worker_runtime::auth::RuntimeIdentityMaterial; - -// The cfg(test) direct Orchestrator adapter seeds its binding for this existing -// fixture Workspace identity. Databases and executors remain per-fixture owners. -const WORKSPACE: &str = "0192f0e8-4d84-7d6e-a000-000000000001"; -const AT: &str = "2026-01-01T00:00:00Z"; -const WATCHDOG: StdDuration = StdDuration::from_secs(10); -const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(5); - -async fn observed(future: impl Future) -> T { - tokio::time::timeout(OBSERVE_LIMIT, future) - .await - .expect("assignment request did not reach the expected SQLite/publication boundary") -} -fn lane(workers: usize, queued: usize) -> DbExecutor { - DbExecutor::new(DbExecutorLimits { - workers: NonZeroUsize::new(workers).unwrap(), - queued: NonZeroUsize::new(queued).unwrap(), - }) - .unwrap() -} - -struct Fixture { - api: WorkspaceApi, - store: Arc, - executor: DbExecutor, - execution: Arc, - ticket: ticket::TicketRef, - _dir: tempfile::TempDir, -} -impl Fixture { - async fn new() -> Self { - let dir = tempfile::tempdir().unwrap(); - let executor = lane(1, 32); - let config = ServerConfig { - workspace_id: WORKSPACE.into(), - workspace_display_name: "Assignment DB fixture".into(), - workspace_created_at: AT.into(), - workspace_execution_root: dir.path().into(), - database_path: dir.path().join("server.db"), - data_root: dir.path().join("data"), - embedded_runtime_store_root: dir.path().join("runtime"), - static_assets_dir: None, - auth: AuthConfig::Passkey { - rp_id: "localhost".into(), - origin: "http://localhost:5173".into(), - public_base_url: "http://localhost:5173".into(), - cookie_name: "assignment_db_session".into(), - }, - max_records: 100, - repositories: Vec::new(), - runtime_event_sources: Vec::new(), - remote_runtime_sources: Vec::new(), - backend_base_url: Some("assignment-db-server".into()), - }; - let store = Arc::new( - SqliteWorkspaceStore::open_for_test(&config.database_path) - .unwrap() - .with_db_executor(executor.clone()), - ); - seed_test_registered_workspace(store.as_ref(), &config) - .await - .unwrap(); - store - .upsert_user(&UserRecord { - user_id: "fixture-user".into(), - account_id: format!("account-{WORKSPACE}"), - handle: "fixture-user".into(), - display_name: "Fixture user".into(), - created_at: AT.into(), - updated_at: AT.into(), - }) - .unwrap(); - store - .create_browser_session(&crate::store::BrowserSessionRecord { - session_id: "fixture-session".into(), - token_hash: crate::auth::token_hash("fixture-browser-token"), - user_id: "fixture-user".into(), - created_at: AT.into(), - expires_at: "2999-01-01T00:00:00Z".into(), - revoked_at: None, - }) - .unwrap(); - store - .create_api_token(&ApiTokenRecord { - token_hash: crate::auth::token_hash("fixture-api-token"), - token_id: "fixture-token".into(), - user_id: "fixture-user".into(), - label: "assignment fixture".into(), - created_at: AT.into(), - expires_at: None, - last_used_at: None, - revoked_at: None, - }) - .unwrap(); - let execution = Arc::new(DeterministicExecutionBackend::default()); - // Explicit, fixture-owned finite lanes; never the production singleton. - let host_executors = HostExecutors::default(); - host_executors - .host - .set(Some(lane(2, 16))) - .unwrap_or_else(|_| panic!("Host owner initialized twice")); - host_executors - .cleanup - .set(Some(lane(2, 16))) - .unwrap_or_else(|_| panic!("cleanup owner initialized twice")); - let api = WorkspaceApi::new_with_execution_backend_and_broker( - config, - store.clone(), - execution.clone(), - BackendResourceBroker::default(), - host_executors, - None, - ) - .await - .unwrap(); - let mut input = ticket::NewTicket::new("Continuing assignment input"); - input.workflow_state = Some(TicketState::Open); - let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); - store - .set_current_ticket_role_assignment( - &TicketRoleAssignmentRecord { - workspace_id: WORKSPACE.into(), - ticket_id: ticket.id.clone(), - assignment_id: "fixture-assignment".into(), - role: TicketAssignmentRole::Orchestrator, - principal: TicketAssignmentPrincipal::WorkspaceAgent { - agent_key: "workspace-orchestrator".into(), - }, - assigned_by: "fixture-user".into(), - assigned_at: AT.into(), - }, - None, - "fixture-assign-event", - "fixture-assign-operation", - false, - ) - .unwrap(); - executor.drain().await.unwrap(); - Self { - api, - store, - executor, - execution, - ticket, - _dir: dir, - } - } - fn path(&self, browser: bool, suffix: &str) -> String { - let prefix = if browser { "api/w" } else { "workspaces" }; - // Exercise reference resolution, not merely an internal row ID. - format!( - "/{prefix}/{WORKSPACE}/tickets/{}/{suffix}", - self.ticket.resource_key.as_deref().unwrap() - ) - } - fn request(method: &str, path: &str, body: serde_json::Value) -> Request { - Request::builder() - .method(method) - .uri(path) - .header(CONTENT_TYPE, "application/json") - .header("cookie", "assignment_db_session=fixture-browser-token") - .header(ORIGIN, "http://localhost:5173") - .body(axum::body::Body::from(body.to_string())) - .unwrap() - } - fn account_request(method: &str, path: &str, body: serde_json::Value) -> Request { - Request::builder() - .method(method) - .uri(path) - .header(CONTENT_TYPE, "application/json") - .header("authorization", "Bearer fixture-api-token") - .body(axum::body::Body::from(body.to_string())) - .unwrap() - } - fn actor() -> RequestActor { - RequestActor { - user_id: "fixture-user".into(), - account_id: format!("account-{WORKSPACE}"), - handle: "fixture-user".into(), - display_name: "Fixture user".into(), - auth_method: ActorAuthMethod::BrowserSession, - } - } - // Contention tests isolate handler SQL from authentication's earlier SQL. - // Full signed, token and browser authentication is tested separately below. - fn inner_router(&self) -> Router { - build_inner_router(self.api.clone()).layer(Extension(Self::actor())) - } - fn answer(&self, key: &str) -> serde_json::Value { - let current = self - .store - .get_ticket_assignment_tracking(WORKSPACE, &self.ticket.id, "fixture-assignment") - .unwrap() - .unwrap(); - serde_json::json!({ - "assignment_id": current.assignment_id, - "expected_last_event_id": current.last_event_id.unwrap(), - "operation_key": key, "body": "Explicit user answer: preserve this exact body" - }) - } - async fn start_orchestrator(&self) -> RuntimeWorkerRef { - // Live Runtime binding is required by the real delivery acknowledger. - let identity = (*EMBEDDED_RUNTIME_REQUEST_IDENTITY).clone(); - self.store - .upsert_workspace_runtime_binding( - WorkspaceRuntimeBinding { - workspace_id: WORKSPACE.into(), - runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), - display_name: "Embedded fixture Runtime".into(), - base_url: "in-process://embedded".into(), - public_key: identity.public_key, - public_key_fingerprint: String::new(), - binding_id: "fixture-binding".into(), - authentication_mode: StoredRuntimeAuthenticationMode::LegacyServerIssuer, - created_at: AT.into(), - updated_at: AT.into(), - revoked_at: None, - }, - false, - ) - .unwrap(); - let started = scoped_start_workspace_orchestrator( - State(self.api.clone()), - AxumPath(ScopedWorkspacePath { - workspace_id: WORKSPACE.into(), - }), - ) - .await - .unwrap() - .0; - let worker = started.worker.unwrap(); - observed(self.api.host_executors.host().unwrap().drain()) - .await - .unwrap(); - observed(self.executor.drain()).await.unwrap(); - self.execution.take_inputs(); - self.execution.take_input_request_ids(); - RuntimeWorkerRef::new(worker.runtime_id, worker.worker_id) - } - async fn drain(&self) { - observed(self.api.host_executors.host().unwrap().drain()) - .await - .unwrap(); - observed(self.executor.drain()).await.unwrap(); - } - async fn finish(&self) { - // Keep the original Tokio driver alive through actual Host/cleanup work, - // then stop projection producers before the final DB barrier. - observed(self.api.host_executors.host().unwrap().drain()) - .await - .unwrap(); - observed(self.api.host_executors.cleanup().unwrap().drain()) - .await - .unwrap(); - observed(self.api.worker_projection.shutdown_async()).await; - observed(self.executor.drain()).await.unwrap(); - } - fn answer_count(&self) -> i64 { - Connection::open(&self.api.config.database_path).unwrap().query_row( - "SELECT count(*) FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2 AND kind='assignment_answer'", - rusqlite::params![WORKSPACE, self.ticket.id], |r| r.get(0)).unwrap() - } - fn assert_saved_answer(&self, key: &str, answer: &serde_json::Value) -> serde_json::Value { - let conn = Connection::open(&self.api.config.database_path).unwrap(); - let (input, source, result): (String, String, String) = conn.query_row( - "SELECT input_json,source_context_json,result_json FROM typed_ticket_assignment_tracking_operations WHERE workspace_id=?1 AND operation_key=?2", - rusqlite::params![WORKSPACE,key], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); - assert_eq!( - serde_json::from_str::(&input).unwrap()["answer"], - *answer - ); - let source: serde_json::Value = serde_json::from_str(&source).unwrap(); - assert_eq!(source["source"]["kind"], "account", "{source}"); - assert_eq!(source["source"]["user_id"], "fixture-user", "{source}"); - let receipt: serde_json::Value = serde_json::from_str(&result).unwrap(); - let (body, author, audit): (String,String,String) = conn.query_row( - "SELECT e.body,e.author,a.value FROM typed_ticket_events e JOIN typed_ticket_event_attributes a ON a.workspace_id=e.workspace_id AND a.ticket_id=e.ticket_id AND a.event_index=e.event_index AND a.key='authenticated_source' WHERE e.workspace_id=?1 AND e.ticket_id=?2 AND e.event_index=?3", - rusqlite::params![WORKSPACE,self.ticket.id,receipt["event_index"].as_u64().unwrap()], - |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); - assert_eq!(body, answer["body"].as_str().unwrap()); - assert_eq!(author, source["actor"].as_str().unwrap()); - assert_eq!( - serde_json::from_str::(&audit).unwrap(), - source - ); - receipt - } -} - -// SQLite owns the actual contention. Watchdog is a panic/hang escape, not test timing. -struct WriteLock { - release: Option>, - thread: Option>, - released: Arc, -} -impl WriteLock { - fn acquire(path: PathBuf) -> Self { - let (ready, held) = mpsc::sync_channel(1); - let (release, permit) = mpsc::channel(); - let released = Arc::new(AtomicBool::new(false)); - let flag = released.clone(); - let thread = std::thread::spawn(move || { - let conn = Connection::open(path).unwrap(); - conn.execute_batch("BEGIN IMMEDIATE").unwrap(); - ready.send(()).unwrap(); - let _ = permit.recv_timeout(WATCHDOG); - conn.execute_batch("ROLLBACK").unwrap(); - flag.store(true, Ordering::SeqCst); - }); - held.recv_timeout(WATCHDOG) - .expect("SQLite blocker did not acquire ownership"); - Self { - release: Some(release), - thread: Some(thread), - released, - } - } - fn release(&mut self) { - if let Some(release) = self.release.take() { - let _ = release.send(()); - } - if let Some(thread) = self.thread.take() { - thread.join().unwrap(); - } - } -} -impl Drop for WriteLock { - fn drop(&mut self) { - self.release(); - } -} - -struct BusyGate { - entered: Option>, - resume: mpsc::Receiver<()>, -} -struct BusyObservation { - store: Arc, - gate: Box, - resume: Option>, -} -impl BusyObservation { - fn install(store: Arc) -> (Self, oneshot::Receiver<()>) { - unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 { - // SAFETY: boxed fixture userdata stays alive until mutex-serialized unregister. - let gate = unsafe { &mut *context.cast::() }; - if let Some(entered) = gate.entered.take() { - let _ = entered.send(()); - return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok()); - } - 0 - } - let (entered, waiting) = oneshot::channel(); - let (resume, permit) = mpsc::channel(); - let mut gate = Box::new(BusyGate { - entered: Some(entered), - resume: permit, - }); - store - .with_conn(|conn| { - // SAFETY: connection owns handle; Box gives userdata a stable address. - assert_eq!( - unsafe { - ffi::sqlite3_busy_handler( - conn.handle(), - Some(busy), - (&mut *gate as *mut BusyGate).cast(), - ) - }, - ffi::SQLITE_OK - ); - Ok(()) - }) - .unwrap(); - ( - Self { - store, - gate, - resume: Some(resume), - }, - waiting, - ) - } - fn resume(&mut self) { - if let Some(resume) = self.resume.take() { - let _ = resume.send(()); - } - } -} -impl Drop for BusyObservation { - fn drop(&mut self) { - self.resume(); - self.store - .with_conn(|conn| { - // SAFETY: unregister before the owned Box is freed. - unsafe { - ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut()); - } - Ok(()) - }) - .unwrap(); - let _ = &self.gate; - } -} -async fn runtime_progress_while_locked(lock: &WriteLock) { - assert_eq!( - observed(tokio::spawn(async { "independent control" })) - .await - .unwrap(), - "independent control" - ); - // This timer IS the responsiveness contract; it does not guess SQL acceptance. - observed(tokio::time::sleep(StdDuration::from_millis(1))).await; - assert!( - !lock.released.load(Ordering::SeqCst), - "watchdog released lock before control/timer progressed" - ); -} -async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value { - let status = response.status(); - let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) - .await - .unwrap(); - assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); - if bytes.is_empty() { - // Authentication middleware deliberately returns status-only denials; - // generated handler/domain successes and errors must still be JSON. - assert!( - matches!(expected, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN), - "unexpected empty HTTP body for {expected}" - ); - return serde_json::Value::Null; - } - serde_json::from_slice(&bytes).expect("typed JSON HTTP response") -} - -// Request lifecycle source and actual retained leases, not waiter liveness. -#[derive(Default)] -struct Lifetimes { - live: Arc, - admissions: AtomicUsize, -} -struct Lease(Arc); -impl Drop for Lease { - fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::SeqCst); - } -} -impl DbOperationLifetime for Lifetimes { - fn retain(&self) -> Box { - self.admissions.fetch_add(1, Ordering::SeqCst); - self.live.fetch_add(1, Ordering::SeqCst); - Box::new(Lease(self.live.clone())) - } -} - -// Finite admitted blockers release through RAII even if an HTTP assertion fails. -struct Saturation { - releases: Vec>, -} -impl Saturation { - fn fill(executor: &DbExecutor, workers: usize, queued: usize) -> Self { - let mut releases = Vec::new(); - for _ in 0..workers { - let (started, running) = mpsc::channel(); - let (release, permit) = mpsc::channel(); - executor - .dispatch(move || { - started.send(()).unwrap(); - let _ = permit.recv_timeout(WATCHDOG); - }) - .unwrap(); - running - .recv_timeout(WATCHDOG) - .expect("admitted blocker did not start"); - releases.push(release); - } - for _ in 0..queued { - let (release, permit) = mpsc::channel(); - executor - .dispatch(move || { - let _ = permit.recv_timeout(WATCHDOG); - }) - .unwrap(); - releases.push(release); - } - Self { releases } - } -} -impl Drop for Saturation { - fn drop(&mut self) { - self.releases.clear(); - } -} - -#[tokio::test(flavor = "current_thread")] -async fn assignment_answer_sqlite_writer_wait_keeps_timer_control_and_receipt_atomic() { - for browser in [false, true] { - let fixture = Fixture::new().await; - let answer = fixture.answer("busy-answer"); - let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); - let (mut busy, waiting) = BusyObservation::install(fixture.store.clone()); - let response = tokio::spawn(fixture.inner_router().oneshot(Fixture::request( - "POST", - &fixture.path(browser, "assignment-answer"), - answer.clone(), - ))); - observed(waiting).await.unwrap(); // actual SQLite busy callback, not first poll - runtime_progress_while_locked(&lock).await; - assert!( - !response.is_finished(), - "answer responded before write ownership" - ); - assert_eq!( - fixture.answer_count(), - 0, - "uncommitted answer became visible" - ); - lock.release(); - busy.resume(); - let receipt = - response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; - fixture.drain().await; - assert_eq!( - fixture.answer_count(), - 1, - "one call commits once; no mutation retries" - ); - assert_eq!(fixture.assert_saved_answer("busy-answer", &answer), receipt); - fixture.finish().await; - } -} - -#[tokio::test(flavor = "current_thread")] -async fn saturated_db_assignment_routes_return_503_without_accepting_answer() { - let fixture = Fixture::new().await; - let answer = fixture.answer("unaccepted-answer"); - let full = Saturation::fill(&fixture.executor, 1, 32); - let lifetime = Arc::new(Lifetimes::default()); - for browser in [false, true] { - for (method, suffix, input) in [ - ( - "GET", - "assignment-tracking/assignments", - serde_json::Value::Null, - ), - ( - "GET", - "assignment-diagnostics?assignment_id=fixture-assignment", - serde_json::Value::Null, - ), - ("POST", "assignment-answer", answer.clone()), - ] { - let response = observed(crate::db_executor::with_operation_lifetime( - lifetime.clone(), - fixture.inner_router().oneshot(Fixture::request( - method, - &fixture.path(browser, suffix), - input, - )), - )) - .await - .unwrap(); - let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; - assert_eq!( - body["diagnostics"][0]["code"], "db_capacity_exceeded", - "{body}" - ); - } - } - let response = observed(fixture.inner_router().oneshot(Fixture::request( - "POST", - &format!("/api/w/{WORKSPACE}/tickets/query"), - serde_json::json!({"handling_status":"waiting"}), - ))) - .await - .unwrap(); - let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; - assert_eq!( - body["diagnostics"][0]["code"], "db_capacity_exceeded", - "{body}" - ); - assert_eq!(lifetime.live.load(Ordering::SeqCst), 0); - drop(full); - fixture.drain().await; - assert_eq!(fixture.answer_count(), 0); - assert_eq!(Connection::open(&fixture.api.config.database_path).unwrap().query_row( - "SELECT count(*) FROM typed_ticket_assignment_tracking_operations WHERE operation_key='unaccepted-answer'", [], |r| r.get::<_,i64>(0)).unwrap(), 0); - fixture.finish().await; -} - -#[tokio::test(flavor = "current_thread")] -async fn cancelled_answer_sql_job_keeps_audit_receipt_publication_and_inherited_lifecycle() { - let fixture = Fixture::new().await; - let orchestrator = fixture.start_orchestrator().await; - let answer = fixture.answer("cancelled-answer"); - let (published, publication) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - fixture.execution.before_input_returns(move |_| { - let _ = published.send(()); - permit - .recv_timeout(WATCHDOG) - .expect("publication was not released"); - }); - let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); - let (mut busy, waiting) = BusyObservation::install(fixture.store.clone()); - let lifetime = Arc::new(Lifetimes::default()); - let response = tokio::spawn(crate::db_executor::with_operation_lifetime( - lifetime.clone(), - fixture.inner_router().oneshot(Fixture::request( - "POST", - &fixture.path(true, "assignment-answer"), - answer.clone(), - )), - )); - observed(waiting).await.unwrap(); - assert!(lifetime.live.load(Ordering::SeqCst) > 0); - response.abort(); - assert!(observed(response).await.unwrap_err().is_cancelled()); - assert!( - lifetime.live.load(Ordering::SeqCst) > 0, - "waiter cancellation released accepted SQL lifecycle" - ); - // Queue capacity remains bounded after waiter loss. - let mut queued_releases: Vec> = Vec::new(); - for _ in 0..32 { - let (queued_release, queued_permit) = mpsc::channel(); - fixture - .executor - .dispatch(move || { - let _ = queued_permit.recv_timeout(WATCHDOG); - }) - .unwrap(); - queued_releases.push(queued_release); - } - assert!(matches!( - fixture.executor.dispatch(|| ()), - Err(Error::DbCapacityExceeded) - )); - lock.release(); - busy.resume(); - drop(queued_releases); - observed(publication).await.unwrap(); // real Runtime dispatch hook reached after SQL commit - observed(fixture.executor.drain()).await.unwrap(); - assert_eq!(fixture.answer_count(), 1); - fixture.assert_saved_answer("cancelled-answer", &answer); - // Runtime wait belongs to Host: the sole DB lane remains available for callbacks. - assert_eq!( - observed( - fixture - .executor - .run(|| Ok::<_, Error>("DB callback progressed")) - ) - .await - .unwrap(), - "DB callback progressed" - ); - assert!( - lifetime.live.load(Ordering::SeqCst) > 0, - "Host publication did not inherit admitted lifecycle" - ); - assert!( - lifetime.admissions.load(Ordering::SeqCst) >= 2, - "no inherited Host admission" - ); - release.send(()).unwrap(); - fixture.drain().await; - assert_eq!( - lifetime.live.load(Ordering::SeqCst), - 0, - "actual cleanup leaked lifecycle" - ); - let inputs = fixture.execution.take_inputs(); - assert_eq!( - inputs.len(), - 1, - "cancelled HTTP answer must still publish exactly once" - ); - assert_eq!(inputs[0].0.worker_id.to_string(), orchestrator.worker_id); - assert!( - inputs[0] - .1 - .contains(fixture.ticket.resource_key.as_deref().unwrap()) - ); - assert!(inputs[0].1.contains("input:")); - assert_eq!(fixture.execution.take_input_request_ids().len(), 1); - fixture.finish().await; -} - -#[tokio::test(flavor = "current_thread")] -async fn assignment_answer_http_success_waits_for_publication_without_holding_db_worker() { - let fixture = Fixture::new().await; - fixture.start_orchestrator().await; - let (published, publication) = oneshot::channel(); - let (release, permit) = mpsc::channel(); - fixture.execution.before_input_returns(move |_| { - let _ = published.send(()); - permit - .recv_timeout(WATCHDOG) - .expect("publication was not released"); - }); - let answer = fixture.answer("await-publication"); - let response = tokio::spawn(build_router(fixture.api.clone()).oneshot(Fixture::request( - "POST", - &fixture.path(true, "assignment-answer"), - answer.clone(), - ))); - observed(publication).await.unwrap(); - assert!( - !response.is_finished(), - "normal HTTP waiter returned before publication completion" - ); - assert_eq!(fixture.answer_count(), 1, "publication must follow commit"); - assert_eq!( - observed(fixture.executor.run(|| Ok::<_, Error>(42))) - .await - .unwrap(), - 42, - "Runtime dispatch blocked the sole DB worker" - ); - release.send(()).unwrap(); - let receipt = response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; - fixture.drain().await; - assert_eq!( - fixture.assert_saved_answer("await-publication", &answer), - receipt - ); - assert_eq!(fixture.execution.take_inputs().len(), 1); - fixture.finish().await; -} - -// WAL readers do not wait for an IMMEDIATE writer. For read-only callers, hold -// the actual Store connection mutex instead: it is part of the same whole DB -// job as reference resolution and the page/query read (not a mock executor job). -struct ConnectionLock { - release: Option>, - thread: Option>, - released: Arc, -} -impl ConnectionLock { - fn acquire(store: Arc) -> Self { - let (ready, held) = mpsc::sync_channel(1); - let (release, permit) = mpsc::channel(); - let released = Arc::new(AtomicBool::new(false)); - let flag = released.clone(); - let thread = std::thread::spawn(move || { - store - .with_conn(|conn| { - assert_eq!(conn.query_row("SELECT 1", [], |r| r.get::<_, i64>(0))?, 1); - ready.send(()).unwrap(); - let _ = permit.recv_timeout(WATCHDOG); - Ok(()) - }) - .unwrap(); - flag.store(true, Ordering::SeqCst); - }); - held.recv_timeout(WATCHDOG) - .expect("Store connection owner did not start"); - Self { - release: Some(release), - thread: Some(thread), - released, - } - } - fn release(&mut self) { - if let Some(release) = self.release.take() { - let _ = release.send(()); - } - if let Some(thread) = self.thread.take() { - thread.join().unwrap(); - } - } -} -impl Drop for ConnectionLock { - fn drop(&mut self) { - self.release(); - } -} -struct Admission(Mutex>>); -impl DbOperationLifetime for Admission { - fn retain(&self) -> Box { - if let Some(entered) = self.0.lock().unwrap().take() { - let _ = entered.send(()); - } - Box::new(()) - } -} - -#[tokio::test(flavor = "current_thread")] -async fn assignment_read_pages_and_reference_resolution_wait_off_executor_on_store_connection() { - let fixture = Fixture::new().await; - for browser in [false, true] { - for suffix in [ - "assignment-tracking/assignments?limit=1", - "assignment-diagnostics?assignment_id=fixture-assignment&limit=1", - ] { - let mut lock = ConnectionLock::acquire(fixture.store.clone()); - let (entered, admission) = oneshot::channel(); - let response = tokio::spawn(crate::db_executor::with_operation_lifetime( - Arc::new(Admission(Mutex::new(Some(entered)))), - fixture.inner_router().oneshot(Fixture::request( - "GET", - &fixture.path(browser, suffix), - serde_json::Value::Null, - )), - )); - observed(admission).await.unwrap(); - assert_eq!(observed(tokio::spawn(async { 42 })).await.unwrap(), 42); - observed(tokio::time::sleep(StdDuration::from_millis(1))).await; - assert!( - !lock.released.load(Ordering::SeqCst), - "synchronous pre-call reference read stalled Tokio until watchdog" - ); - assert!( - !response.is_finished(), - "read bypassed held Store connection" - ); - lock.release(); - let body = - response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; - if suffix.starts_with("assignment-tracking") { - assert_eq!( - body["items"][0]["assignment_id"], "fixture-assignment", - "{body}" - ); - assert!(body["next_assignment_id"].is_null(), "{body}"); - } else { - assert_eq!(body["assignment_id"], "fixture-assignment", "{body}"); - assert_eq!(body["input_origin_known"], true, "{body}"); - assert!(body.get("payload_json").is_none(), "{body}"); - } - fixture.drain().await; - } - } - fixture.finish().await; -} - -#[tokio::test(flavor = "current_thread")] -async fn typed_ticket_query_offloads_principal_handling_and_closure_filters_before_page_limit() { - let fixture = Fixture::new().await; - let backend = browser_ticket_backend(&fixture.api).unwrap(); - let closed = backend - .create(ticket::NewTicket::new("Closed unrelated case")) - .unwrap(); - let before = backend.show(closed.id.clone().into()).unwrap(); - backend - .update_case( - &closed.id, - ticket::TicketCaseUpdate { - expected_content_digest: ticket::ticket_content_digest(&before), - expected_state: before.meta.workflow_state, - expected_closure_operation_key: before.closure_operation_key.clone(), - operation_key: "query-close".into(), - action: ticket::TicketCaseAction::Close { - reason: ticket::TicketClosureReason::Completed, - }, - explanation: "Confirmed result".into(), - references: vec![], - author: None, - }, - ) - .unwrap(); - for n in 0..3 { - backend - .create(ticket::NewTicket::new(format!("Newer unassigned {n}"))) - .unwrap(); - } - let path = format!("/api/w/{WORKSPACE}/tickets/query"); - for (input, expected) in [ - ( - serde_json::json!({"assigned_principal":{"kind":"workspace_agent","agent_key":"workspace-orchestrator"},"handling_status":"waiting","limit":1}), - Some(fixture.ticket.id.as_str()), - ), - ( - serde_json::json!({"assigned_principal":{"kind":"workspace_agent","agent_key":"workspace-orchestrator"},"handling_status":"done","limit":1}), - None, - ), - ( - serde_json::json!({"assigned_principal":{"kind":"worker","runtime_id":"unavailable","worker_id":"unavailable"},"limit":1}), - None, - ), - ( - serde_json::json!({"closure_reason":"completed","limit":1}), - Some(closed.id.as_str()), - ), - ( - serde_json::json!({"closure_reason":"completed","handling_status":"waiting","limit":1}), - None, - ), - ] { - // Candidate SQL has its own Authority connection, including empty-result - // queries that never reach API Store enrichment. - let mut lock = ConnectionLock::acquire(Arc::new( - fixture.api.authority.candidate_query_store_for_test(), - )); - let (entered, admission) = oneshot::channel(); - let response = tokio::spawn(crate::db_executor::with_operation_lifetime( - Arc::new(Admission(Mutex::new(Some(entered)))), - fixture - .inner_router() - .oneshot(Fixture::request("POST", &path, input)), - )); - observed(admission).await.unwrap(); - assert_eq!( - observed(tokio::spawn(async { "query control" })) - .await - .unwrap(), - "query control" - ); - observed(tokio::time::sleep(StdDuration::from_millis(1))).await; - assert!( - !lock.released.load(Ordering::SeqCst), - "query blocked Tokio instead of its admitted DB worker" - ); - if response.is_finished() { - lock.release(); - let response = observed(response).await.unwrap().unwrap(); - let status = response.status(); - let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) - .await - .unwrap(); - panic!( - "query returned before Store connection release: status={status}, body={}", - String::from_utf8_lossy(&body) - ); - } - lock.release(); - let body = response_json(observed(response).await.unwrap().unwrap(), StatusCode::OK).await; - let items = body["items"].as_array().unwrap(); - if let Some(id) = expected { - assert_eq!(items.len(), 1, "{body}"); - assert_eq!(items[0]["id"], id, "{body}"); - } else { - assert!(items.is_empty(), "{body}"); - } - fixture.drain().await; - } - // Also cross the complete browser and Account authentication boundaries. - for request in [ - Fixture::request( - "POST", - &path, - serde_json::json!({"handling_status":"waiting"}), - ), - Fixture::account_request( - "POST", - &path, - serde_json::json!({"closure_reason":"completed"}), - ), - ] { - let body = response_json( - observed(build_router(fixture.api.clone()).oneshot(request)) - .await - .unwrap(), - StatusCode::OK, - ) - .await; - assert_eq!(body["items"].as_array().unwrap().len(), 1, "{body}"); - } - fixture.finish().await; -} - -#[tokio::test(flavor = "current_thread")] -async fn browser_and_account_assignment_routes_preserve_receipt_replay_and_map_cas_cursor_errors() { - let fixture = Fixture::new().await; - let app = build_router(fixture.api.clone()); - let answer = fixture.answer("http-replay"); - let mut saved = None; - for browser in [false, true] { - for (suffix, input) in [ - ( - "assignment-tracking/assignments?limit=1", - serde_json::Value::Null, - ), - ( - "assignment-diagnostics?assignment_id=fixture-assignment&limit=1", - serde_json::Value::Null, - ), - ] { - let path = fixture.path(browser, suffix); - let body = response_json( - observed(app.clone().oneshot(Fixture::request("GET", &path, input))) - .await - .unwrap(), - StatusCode::OK, - ) - .await; - assert!(!body.is_null()); - } - let path = fixture.path(browser, "assignment-answer"); - let response = observed(app.clone().oneshot(Fixture::request( - "POST", - &path, - answer.clone(), - ))) - .await - .unwrap(); - let receipt = response_json(response, StatusCode::OK).await; - if let Some(saved) = &saved { - assert_eq!(&receipt, saved); - } else { - saved = Some(receipt); - } - let receipt = response_json( - observed( - app.clone() - .oneshot(Fixture::account_request("POST", &path, answer.clone())), - ) - .await - .unwrap(), - StatusCode::OK, - ) - .await; - assert_eq!( - Some(receipt), - saved, - "same authenticated Account/User through token and cookie must replay" - ); - let mut changed = answer.clone(); - changed["body"] = serde_json::json!("different replay body"); - let error = response_json( - observed( - app.clone() - .oneshot(Fixture::request("POST", &path, changed)), - ) - .await - .unwrap(), - StatusCode::CONFLICT, - ) - .await; - assert!(error.to_string().contains("fingerprint"), "{error}"); - let mut stale = answer.clone(); - stale["operation_key"] = serde_json::json!(format!("stale-{browser}")); - stale["expected_last_event_id"] = serde_json::json!("unobserved-event"); - response_json( - observed(app.clone().oneshot(Fixture::request("POST", &path, stale))) - .await - .unwrap(), - StatusCode::CONFLICT, - ) - .await; - for suffix in [ - "assignment-tracking/assignments?after_assignment_id=foreign", - "assignment-diagnostics?assignment_id=fixture-assignment&after_input_event_id=ticket-event:999", - "assignment-diagnostics?assignment_id=fixture-assignment&limit=101", - ] { - response_json( - observed(app.clone().oneshot(Fixture::request( - "GET", - &fixture.path(browser, suffix), - serde_json::Value::Null, - ))) - .await - .unwrap(), - StatusCode::BAD_REQUEST, - ) - .await; - } - let mut without_origin = Fixture::request("POST", &path, answer.clone()); - without_origin.headers_mut().remove(ORIGIN); - response_json( - observed(app.clone().oneshot(without_origin)).await.unwrap(), - StatusCode::FORBIDDEN, - ) - .await; - let missing = path.replace( - fixture.ticket.resource_key.as_deref().unwrap(), - "T-999999999", - ); - response_json( - observed( - app.clone() - .oneshot(Fixture::request("POST", &missing, answer.clone())), - ) - .await - .unwrap(), - StatusCode::NOT_FOUND, - ) - .await; - let mut unauthenticated = Fixture::request("POST", &path, answer.clone()); - unauthenticated.headers_mut().remove("cookie"); - response_json( - observed(app.clone().oneshot(unauthenticated)) - .await - .unwrap(), - StatusCode::UNAUTHORIZED, - ) - .await; - } - fixture.drain().await; - assert_eq!(fixture.answer_count(), 1); - assert_eq!( - Some(fixture.assert_saved_answer("http-replay", &answer)), - saved - ); - fixture.finish().await; -} - -// Uses the same request-proof signer and registered source records as existing -// HTTP auth fixtures. Runtime identity comes only from the verified proof. -fn signed_request( - identity: &RuntimeIdentityMaterial, - worker: &str, - method: &str, - path: &str, - body: serde_json::Value, -) -> Request { - let body = if method == "GET" { - Vec::new() - } else { - serde_json::to_vec(&body).unwrap() - }; - let proof = worker_runtime::auth::RuntimeRequestSourceSigner::from_identity(identity) - .issue( - "assignment-db-server", - WORKSPACE, - Some(worker), - worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION, - method, - path, - &body, - i64::try_from(worker_runtime::auth::unix_now_seconds()).unwrap(), - 30, - ) - .unwrap(); - Request::builder() - .method(method) - .uri(path) - .header(CONTENT_TYPE, "application/json") - .header( - worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER, - proof, - ) - .body(axum::body::Body::from(body)) - .unwrap() -} - -#[tokio::test(flavor = "current_thread")] -async fn signed_assignment_answer_rejects_non_orchestrator_and_replayed_proof_without_a_receipt() { - let fixture = Fixture::new().await; - let identity = RuntimeIdentityMaterial::generate("signed-source").unwrap(); - fixture - .store - .upsert_workspace_runtime_binding( - WorkspaceRuntimeBinding { - workspace_id: WORKSPACE.into(), - runtime_id: "signed-source".into(), - display_name: "Signed source".into(), - base_url: "https://runtime.example.invalid".into(), - public_key: identity.public_key.clone(), - public_key_fingerprint: String::new(), - binding_id: "signed-binding".into(), - authentication_mode: StoredRuntimeAuthenticationMode::WorkspaceIdentity, - created_at: AT.into(), - updated_at: AT.into(), - revoked_at: None, - }, - true, - ) - .unwrap(); - fixture - .store - .upsert_worker_registry(&WorkerRegistryRecord { - workspace_id: WORKSPACE.into(), - worker: RuntimeWorkerRef::new("signed-source", "source-worker"), - display_name: "Source worker".into(), - profile: Some("builtin:coder".into()), - retention_state: "normal".into(), - transcript_ref: None, - session_ref: None, - summary_ref: None, - diagnostics_ref: None, - created_at: AT.into(), - updated_at: AT.into(), - }) - .unwrap(); - let app = build_router(fixture.api.clone()); - for browser in [false, true] { - let path = fixture.path(browser, "assignment-answer"); - let answer = fixture.answer("denied-source"); - let request = signed_request(&identity, "source-worker", "POST", &path, answer.clone()); - let proof = - request.headers()[worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER].clone(); - response_json( - observed(app.clone().oneshot(request)).await.unwrap(), - StatusCode::FORBIDDEN, - ) - .await; - let mut replay = signed_request(&identity, "source-worker", "POST", &path, answer.clone()); - replay.headers_mut().insert( - worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER, - proof, - ); - response_json( - observed(app.clone().oneshot(replay)).await.unwrap(), - StatusCode::UNAUTHORIZED, - ) - .await; - // The identical registered signed Worker can read; 403 above is domain - // authorization, not accidentally failed proof authentication. - for suffix in [ - "assignment-tracking/assignments?limit=1", - "assignment-diagnostics?assignment_id=fixture-assignment", - ] { - response_json( - observed(app.clone().oneshot(signed_request( - &identity, - "source-worker", - "GET", - &fixture.path(browser, suffix), - serde_json::Value::Null, - ))) - .await - .unwrap(), - StatusCode::OK, - ) - .await; - } - } - fixture.drain().await; - assert_eq!(fixture.answer_count(), 0); - assert_eq!(Connection::open(&fixture.api.config.database_path).unwrap().query_row( - "SELECT count(*) FROM typed_ticket_assignment_tracking_operations WHERE operation_key='denied-source'",[],|r|r.get::<_,i64>(0)).unwrap(),0); - fixture.finish().await; -} - -#[tokio::test(flavor = "current_thread")] -async fn registered_current_signed_orchestrator_answers_both_routes_with_source_bound_receipt() { - let fixture = Fixture::new().await; - let orchestrator = fixture.start_orchestrator().await; - let identity = (*EMBEDDED_RUNTIME_REQUEST_IDENTITY).clone(); - let binding_id = fixture - .api - .config_store - .get_workspace_runtime_binding(WORKSPACE, EMBEDDED_WORKER_RUNTIME_ID) - .unwrap() - .unwrap() - .binding_id; - let answer = fixture.answer("signed-current-answer"); - let mut saved = None; - let app = build_router(fixture.api.clone()); - for browser in [false, true] { - let request = signed_request( - &identity, - &orchestrator.worker_id, - "POST", - &fixture.path(browser, "assignment-answer"), - answer.clone(), - ); - let receipt = response_json( - observed(app.clone().oneshot(request)).await.unwrap(), - StatusCode::OK, - ) - .await; - if let Some(saved) = &saved { - assert_eq!(&receipt, saved); - } else { - saved = Some(receipt); - } - } - fixture.drain().await; - assert_eq!(fixture.answer_count(), 1); - assert_eq!( - fixture.execution.take_inputs().len(), - 1, - "receipt replay must not republish input" - ); - let conn = Connection::open(&fixture.api.config.database_path).unwrap(); - let source: String = conn.query_row( - "SELECT source_context_json FROM typed_ticket_assignment_tracking_operations WHERE workspace_id=?1 AND operation_key='signed-current-answer'", - [WORKSPACE],|r|r.get(0)).unwrap(); - let source: serde_json::Value = serde_json::from_str(&source).unwrap(); - assert_eq!(source["source"]["kind"], "runtime", "{source}"); - assert_eq!( - source["source"]["worker_id"], orchestrator.worker_id, - "{source}" - ); - assert_eq!( - source["source"]["authentication_binding_id"], binding_id, - "{source}" - ); - // Same operation key/body from an Account is not the signed Worker's receipt. - response_json( - observed(app.oneshot(Fixture::account_request( - "POST", - &fixture.path(true, "assignment-answer"), - answer, - ))) - .await - .unwrap(), - StatusCode::CONFLICT, - ) - .await; - assert_eq!(fixture.answer_count(), 1); - fixture.finish().await; -} - -#[tokio::test(flavor = "current_thread")] -async fn saturated_signed_assignment_live_discovery_returns_503_before_sql_while_db_callbacks_progress() - { - let fixture = Fixture::new().await; - let orchestrator = fixture.start_orchestrator().await; - let binding_id = fixture - .api - .config_store - .get_workspace_runtime_binding(WORKSPACE, &orchestrator.runtime_id) - .unwrap() - .unwrap() - .binding_id; - let context = server_api::ServerRequestContext { - actor: None, - worker_source: Some(server_api::ServerWorkerSource { - runtime_id: orchestrator.runtime_id.clone(), - worker_id: orchestrator.worker_id.clone(), - }), - runtime_source: Some(server_api::ServerRuntimeSource { - authentication_binding_id: binding_id, - runtime_id: orchestrator.runtime_id.clone(), - worker_id: Some(orchestrator.worker_id.clone()), - }), - origin: None, - transport_headers: Vec::new(), - }; - // Isolate the generated handler's live-discovery admission from earlier - // authentication Host permits. Actual signed proof authentication is covered - // by the current-source/denial tests above; this context is the trusted input - // to the same canonical/Browser generated HTTP handlers. - let service = Arc::new(ServerApiContractService::Workspace(fixture.api.clone())); - let app = server_api::server_api_axum::ticket_assignment_answer(service.clone()) - .merge(server_api::server_api_axum::ticket_assignment_answer_browser(service)) - .layer(Extension(context)); - let full = Saturation::fill(&fixture.api.host_executors.host().unwrap(), 2, 16); - for browser in [false, true] { - let key = format!("host-denied-answer-{browser}"); - let lifetime = Arc::new(Lifetimes::default()); - let response = observed(crate::db_executor::with_operation_lifetime( - lifetime.clone(), - app.clone().oneshot(Fixture::request( - "POST", - &fixture.path(browser, "assignment-answer"), - fixture.answer(&key), - )), - )) - .await - .unwrap(); - let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; - assert_eq!( - body["diagnostics"][0]["code"], "db_capacity_exceeded", - "{body}" - ); - assert_eq!( - lifetime.admissions.load(Ordering::SeqCst), - 1, - "rejected live discovery must not proceed to SQL admission" - ); - assert_eq!( - lifetime.live.load(Ordering::SeqCst), - 0, - "rejected discovery retained a request lifecycle lease" - ); - assert_eq!( - observed( - fixture - .executor - .run(|| Ok::<_, Error>("DB callback progressed")) - ) - .await - .unwrap(), - "DB callback progressed", - "Host saturation consumed the sole DB worker" - ); - assert_eq!( - fixture.answer_count(), - 0, - "discovery rejection committed an answer/audit" - ); - assert_eq!(Connection::open(&fixture.api.config.database_path).unwrap().query_row( - "SELECT count(*) FROM typed_ticket_assignment_tracking_operations WHERE workspace_id=?1 AND operation_key=?2", - rusqlite::params![WORKSPACE, key], |r| r.get::<_, i64>(0)).unwrap(), 0, - "discovery rejection committed a receipt"); - assert!( - fixture.execution.take_inputs().is_empty(), - "unaccepted answer dispatched Runtime input" - ); - } - drop(full); - fixture.finish().await; -} diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index e43bac8fe..09a50e532 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -769,7 +769,7 @@ async fn delivery_real_fixture( DeliveryScriptedClient, ) { let config = test_server_config(root); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) .await .unwrap(); @@ -1007,7 +1007,7 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi tokio::task::yield_now().await; let reopened = WorkspaceApi::new_with_execution_backend( config.clone(), - Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()), + Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()), execution.clone(), ) .await diff --git a/crates/workspace-server/src/server_db_execution_tests.rs b/crates/workspace-server/src/server_db_execution_tests.rs deleted file mode 100644 index e9a6538f2..000000000 --- a/crates/workspace-server/src/server_db_execution_tests.rs +++ /dev/null @@ -1,769 +0,0 @@ -//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs). -use super::*; -use crate::SqliteWorkspaceStore; -use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime}; -use rusqlite::{Connection, ffi}; -use std::{ - ffi::c_void, - future::Future, - num::NonZeroUsize, - sync::{atomic::AtomicBool, mpsc}, - thread::JoinHandle, - time::Duration as StdDuration, -}; -use tokio::sync::oneshot; - -const WORKSPACE: &str = "db-execution"; -const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects"; -const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2); -const WATCHDOG: StdDuration = StdDuration::from_secs(5); - -async fn observed(future: impl Future) -> T { - tokio::time::timeout(OBSERVE_LIMIT, future) - .await - .expect("request did not reach the expected DB/lifecycle state") -} - -struct Fixture { - // Keep temporary authority files alive until all other fixture owners drop. - server: WorkspaceServerApi, - api: WorkspaceApi, - executor: DbExecutor, - _dir: tempfile::TempDir, -} - -impl Fixture { - async fn new() -> Self { - let dir = tempfile::tempdir().unwrap(); - let executor = DbExecutor::new(DbExecutorLimits { - workers: NonZeroUsize::new(1).unwrap(), - queued: NonZeroUsize::new(1).unwrap(), - }) - .unwrap(); - // Explicit startup config: never resolve the developer's live data dir. - let config = ServerConfig { - workspace_id: WORKSPACE.into(), - workspace_display_name: "DB execution fixture".into(), - workspace_created_at: "2026-01-01T00:00:00Z".into(), - workspace_execution_root: dir.path().into(), - database_path: dir.path().join("server.db"), - data_root: dir.path().join("data"), - embedded_runtime_store_root: dir.path().join("runtime"), - static_assets_dir: None, - auth: AuthConfig::Passkey { - rp_id: "localhost".into(), - origin: "http://localhost:5173".into(), - public_base_url: "http://localhost:5173".into(), - cookie_name: "db_execution_session".into(), - }, - max_records: 100, - repositories: Vec::new(), - runtime_event_sources: Vec::new(), - remote_runtime_sources: Vec::new(), - backend_base_url: Some("http://localhost:8787".into()), - }; - let store = Arc::new( - SqliteWorkspaceStore::open_for_test(&config.database_path) - .unwrap() - .with_db_executor(executor.clone()), - ); - seed_test_registered_workspace(store.as_ref(), &config) - .await - .unwrap(); - store - .upsert_user(&UserRecord { - user_id: "fixture-user".into(), - account_id: format!("account-{WORKSPACE}"), - handle: "fixture-user".into(), - display_name: "Fixture user".into(), - created_at: config.workspace_created_at.clone(), - updated_at: config.workspace_created_at.clone(), - }) - .unwrap(); - store - .create_browser_session(&crate::store::BrowserSessionRecord { - session_id: "fixture-session".into(), - token_hash: crate::auth::token_hash("fixture-browser-token"), - user_id: "fixture-user".into(), - created_at: config.workspace_created_at.clone(), - expires_at: "2999-01-01T00:00:00Z".into(), - revoked_at: None, - }) - .unwrap(); - let api = WorkspaceApi::new_for_test(config.clone(), store.clone()) - .await - .unwrap(); - let server = WorkspaceServerApi::new_for_test(config, store); - // Cache only the real inner router; avoid starting the unrelated hook. - server - .apis - .lock() - .await - .insert(WORKSPACE.into(), api.clone()); - server - .routers - .lock() - .await - .insert(WORKSPACE.into(), build_inner_router(api.clone())); - executor.drain().await.unwrap(); - Self { - server, - api, - executor, - _dir: dir, - } - } - - fn request(method: &str, path: &str, body: serde_json::Value) -> Request { - Request::builder() - .method(method) - .uri(path) - .header(CONTENT_TYPE, "application/json") - .header("cookie", "db_execution_session=fixture-browser-token") - .header(ORIGIN, "http://localhost:5173") - .body(axum::body::Body::from(body.to_string())) - .unwrap() - } - - fn subject_request(role: &str) -> Request { - Self::request( - "POST", - SUBJECTS, - serde_json::json!({ "role": role, "behavior_md": "" }), - ) - } - - fn dispatch(&self, request: Request) -> tokio::task::JoinHandle { - let server = self.server.clone(); - tokio::spawn(async move { dispatch_workspace_request(State(server), request).await }) - } - - // A pre-authenticated generated router isolates handler admission from the - // earlier browser-session DB read. Full authentication/dispatch is exercised - // separately by the cancellation and warm-Feature tests below. - fn inner_router(&self) -> Router { - build_inner_router(self.api.clone()).layer(Extension(RequestActor { - user_id: "fixture-user".into(), - account_id: format!("account-{WORKSPACE}"), - handle: "fixture-user".into(), - display_name: "Fixture user".into(), - auth_method: ActorAuthMethod::BrowserSession, - })) - } - - fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) { - let database = self - .api - .feature_storage - .open(&self.api.subjektiv_registration) - .unwrap(); - // Initialize the trusted scope, then obtain SQLite's actual file path - // from the managed handle rather than guessing the Feature directory. - crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap(); - let path = database - .with_connection(|conn| { - Ok(PathBuf::from(conn.query_row( - "SELECT file FROM pragma_database_list WHERE name = 'main'", - [], - |row| row.get::<_, String>(0), - )?)) - }) - .unwrap(); - (database, path) - } -} - -/// SQLite write ownership lives on an OS thread, including a bounded watchdog. -/// Drop releases and joins it even when an assertion panics on the async side. -struct WriteLock { - release: Option>, - thread: Option>, - released: Arc, -} - -impl WriteLock { - fn acquire(path: PathBuf) -> Self { - let (ready, held) = mpsc::sync_channel(1); - let (release, permit) = mpsc::channel(); - let released = Arc::new(AtomicBool::new(false)); - let flag = released.clone(); - let thread = std::thread::spawn(move || { - let conn = Connection::open(path).unwrap(); - conn.execute_batch("BEGIN IMMEDIATE").unwrap(); - ready.send(()).unwrap(); - let _ = permit.recv_timeout(WATCHDOG); - conn.execute_batch("ROLLBACK").unwrap(); - flag.store(true, Ordering::SeqCst); - }); - held.recv_timeout(WATCHDOG) - .expect("SQLite lock holder did not acquire its transaction"); - Self { - release: Some(release), - thread: Some(thread), - released, - } - } - - fn release(&mut self) { - if let Some(release) = self.release.take() { - let _ = release.send(()); - } - if let Some(thread) = self.thread.take() { - thread.join().unwrap(); - } - } -} -impl Drop for WriteLock { - fn drop(&mut self) { - self.release(); - } -} - -struct BusyGate { - entered: Option>, - resume: mpsc::Receiver<()>, -} - -/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's -/// userdata API can, without process globals or changing production hooks. -enum BusyDatabase { - Feature(crate::FeatureDatabase), - Store(Arc), -} -impl BusyDatabase { - fn access(&self, access: impl FnOnce(&Connection)) { - match self { - Self::Feature(database) => database - .with_connection(|conn| { - access(conn); - Ok(()) - }) - .unwrap(), - Self::Store(store) => store - .with_conn(|conn| { - access(conn); - Ok(()) - }) - .unwrap(), - } - } -} -struct BusyObservation { - database: BusyDatabase, - gate: Box, - resume: Option>, -} -impl BusyObservation { - fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) { - Self::install_on(BusyDatabase::Feature(database)) - } - fn install_store(store: Arc) -> (Self, oneshot::Receiver<()>) { - Self::install_on(BusyDatabase::Store(store)) - } - fn install_on(database: BusyDatabase) -> (Self, oneshot::Receiver<()>) { - unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 { - // SAFETY: owned Box remains registered until Drop unregisters it, - // serialized by the managed connection mutex. - let gate = unsafe { &mut *context.cast::() }; - if let Some(entered) = gate.entered.take() { - let _ = entered.send(()); - return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok()); - } - 0 - } - let (entered, waiting) = oneshot::channel(); - let (resume, permit) = mpsc::channel(); - let mut gate = Box::new(BusyGate { - entered: Some(entered), - resume: permit, - }); - database.access(|conn| { - // SAFETY: handle belongs to this connection; the Box is stable. - let code = unsafe { - ffi::sqlite3_busy_handler( - conn.handle(), - Some(busy), - (&mut *gate as *mut BusyGate).cast(), - ) - }; - assert_eq!(code, ffi::SQLITE_OK); - }); - ( - Self { - database, - gate, - resume: Some(resume), - }, - waiting, - ) - } - fn resume(&mut self) { - if let Some(resume) = self.resume.take() { - let _ = resume.send(()); - } - } -} -impl Drop for BusyObservation { - fn drop(&mut self) { - self.resume(); - self.database.access(|conn| { - // SAFETY: unregister before the boxed userdata is freed. - unsafe { - ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut()); - } - }); - let _ = &self.gate; - } -} - -// A handler with one DB stage can expose its admission without adding a test -// hook to production. Polling its future to Pending proves run accepted it. -struct AdmissionObservation(Mutex>>); -impl DbOperationLifetime for AdmissionObservation { - fn retain(&self) -> Box { - if let Some(accepted) = self.0.lock().unwrap().take() { - let _ = accepted.send(()); - } - Box::new(()) - } -} -fn admission_observation() -> (Arc, oneshot::Receiver<()>) { - let (accepted, receiver) = oneshot::channel(); - ( - Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))), - receiver, - ) -} - -struct StateCheckSaturation { - executor: DbExecutor, - stages: std::sync::atomic::AtomicUsize, - releases: Arc>>>, -} -impl DbOperationLifetime for StateCheckSaturation { - fn retain(&self) -> Box { - if self.stages.fetch_add(1, Ordering::SeqCst) == 1 { - // Authentication has completed; saturate just before the lifecycle - // lookup is admitted. The callback's own dispatch admissions are - // distinct stages and do not recurse into this branch. - let (started, running) = mpsc::channel(); - let (release, permit) = mpsc::channel(); - self.executor - .dispatch(move || { - let _ = started.send(()); - let _ = permit.recv_timeout(WATCHDOG); - }) - .unwrap(); - running - .recv_timeout(WATCHDOG) - .expect("state-check blocker did not start"); - self.releases.lock().unwrap().push(release); - let (release, permit) = mpsc::channel(); - self.executor - .dispatch(move || { - let _ = permit.recv_timeout(WATCHDOG); - }) - .unwrap(); - self.releases.lock().unwrap().push(release); - } - Box::new(()) - } -} -struct ReleaseStateCheckBlockers(Arc>>>); -impl Drop for ReleaseStateCheckBlockers { - fn drop(&mut self) { - self.0.lock().unwrap().clear(); - } -} - -#[tokio::test(flavor = "current_thread")] -async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() { - let fixture = Fixture::new().await; - let releases = Arc::new(Mutex::new(Vec::new())); - let release = ReleaseStateCheckBlockers(releases.clone()); - let source = Arc::new(StateCheckSaturation { - executor: fixture.executor.clone(), - stages: std::sync::atomic::AtomicUsize::new(0), - releases, - }); - let router = Router::new() - .route( - SUBJECTS, - axum::routing::post(|| async { StatusCode::NO_CONTENT }), - ) - .layer(axum::middleware::from_fn_with_state( - fixture.api.clone(), - authorize_workspace_api_request, - )); - let response = observed(crate::db_executor::with_operation_lifetime( - source, - router.oneshot(Fixture::subject_request("not-executed")), - )) - .await - .unwrap(); - drop(release); - observed(fixture.executor.drain()).await.unwrap(); - let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; - assert!( - !body.to_string().contains("Workspace is deleting"), - "{body}" - ); -} - -async fn runtime_progress_while_locked(lock: &WriteLock) { - let control = tokio::spawn(async { "control progressed" }); - assert_eq!(observed(control).await.unwrap(), "control progressed"); - // The timer is the contract, not a guessed wait for request completion. - observed(tokio::time::sleep(StdDuration::from_millis(1))).await; - assert!( - !lock.released.load(Ordering::SeqCst), - "SQLite watchdog released the lock before runtime progress" - ); -} -async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value { - let status = response.status(); - let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) - .await - .unwrap(); - assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); - serde_json::from_slice(&bytes).expect("typed JSON response") -} -fn subject_count(path: &Path, role: &str) -> i64 { - Connection::open(path) - .unwrap() - .query_row( - "SELECT count(*) FROM subjects WHERE role = ?1", - [role], - |row| row.get(0), - ) - .unwrap() -} - -#[tokio::test(flavor = "current_thread")] -async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() { - let fixture = Fixture::new().await; - let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); - let (lifetime, accepted) = admission_observation(); - let request = Fixture::request( - "POST", - "/api/w/db-execution/tickets", - serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(), - ); - let mut response = Box::pin(crate::db_executor::with_operation_lifetime( - lifetime, - fixture.inner_router().oneshot(request), - )); - assert!(futures::poll!(&mut response).is_pending()); - observed(accepted).await.unwrap(); - runtime_progress_while_locked(&lock).await; - assert!( - futures::poll!(&mut response).is_pending(), - "mutation responded before the SQLite lock was released" - ); - let reader = Connection::open(&fixture.api.config.database_path).unwrap(); - let count: i64 = reader - .query_row( - "SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1", - [WORKSPACE], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(count, 0, "Ticket must not be visible before commit"); - lock.release(); - let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; - assert!(body["id"].is_string(), "{body}"); - // Ticket dispatch can call Runtime callbacks, so its operation owner is - // the separate bounded host executor, not the store executor. - observed(fixture.api.host_executors.host().unwrap().drain()) - .await - .unwrap(); - fixture.executor.drain().await.unwrap(); - let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap(); - assert_eq!(count, 1, "one accepted HTTP mutation must commit once"); -} - -#[tokio::test(flavor = "current_thread")] -async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() { - let fixture = Fixture::new().await; - let (database, path) = fixture.feature_database(); - drop(database); // FeatureStorage retains Weak handles: the route must cold-open. - let mut lock = WriteLock::acquire(path); - let (lifetime, accepted) = admission_observation(); - let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null); - let mut response = Box::pin(crate::db_executor::with_operation_lifetime( - lifetime, - fixture.inner_router().oneshot(request), - )); - assert!(futures::poll!(&mut response).is_pending()); - observed(accepted).await.unwrap(); - runtime_progress_while_locked(&lock).await; - assert!( - futures::poll!(&mut response).is_pending(), - "cold open must wait for its SQLite scope transaction" - ); - lock.release(); - let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; - assert_eq!(body["items"], serde_json::json!([]), "{body}"); - fixture.executor.drain().await.unwrap(); -} - -#[tokio::test(flavor = "current_thread")] -async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() { - let fixture = Fixture::new().await; - let (database, path) = fixture.feature_database(); - let mut lock = WriteLock::acquire(path.clone()); - let (mut busy, waiting) = BusyObservation::install(database); - let response = fixture.dispatch(Fixture::subject_request("locked-subject")); - observed(waiting).await.unwrap(); - runtime_progress_while_locked(&lock).await; - assert!( - !response.is_finished(), - "HTTP response must not publish an uncommitted subject" - ); - assert_eq!(subject_count(&path, "locked-subject"), 0); - lock.release(); - busy.resume(); - let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await; - assert_eq!(body["role"], "locked-subject", "{body}"); - assert!( - body["memory_fingerprint"].is_string(), - "response assembly must finish in the DB operation: {body}" - ); - fixture.executor.drain().await.unwrap(); - assert_eq!(subject_count(&path, "locked-subject"), 1); -} - -#[tokio::test(flavor = "current_thread")] -async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() { - let fixture = Fixture::new().await; - let (database, path) = fixture.feature_database(); - let mut lock = WriteLock::acquire(path.clone()); - let (mut busy, waiting) = BusyObservation::install(database); - let response = fixture.dispatch(Fixture::subject_request("cancelled-subject")); - observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling. - response.abort(); - assert!(observed(response).await.unwrap_err().is_cancelled()); - let admission = fixture.server.admission_for_workspace(WORKSPACE).await; - let mut deletion = Box::pin(admission.lock_deletion()); - assert!( - futures::poll!(&mut deletion).is_pending(), - "aborting HTTP must not drop the accepted SQL RequestLease" - ); - assert!( - admission.admit(false).is_none(), - "deletion must fence fresh requests" - ); - assert_eq!(subject_count(&path, "cancelled-subject"), 0); - lock.release(); - busy.resume(); - fixture.executor.drain().await.unwrap(); - let deletion = observed(deletion).await; - assert_eq!( - subject_count(&path, "cancelled-subject"), - 1, - "cancelled waiter must neither cancel nor retry accepted SQL" - ); - drop(deletion); - assert!( - admission.admit(false).is_some(), - "deletion lease cleanup must reopen admission" - ); -} - -#[tokio::test(flavor = "current_thread")] -async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() { - let fixture = Fixture::new().await; - let (database, path) = fixture.feature_database(); - let mut lock = WriteLock::acquire(path.clone()); - let (mut busy, waiting) = BusyObservation::install(database); - let running = fixture.dispatch(Fixture::subject_request("running-subject")); - observed(waiting).await.unwrap(); - let (lifetime, accepted) = admission_observation(); - let mut queued = Box::pin(crate::db_executor::with_operation_lifetime( - lifetime, - fixture - .inner_router() - .oneshot(Fixture::subject_request("queued-subject")), - )); - assert!(futures::poll!(&mut queued).is_pending()); - observed(accepted).await.unwrap(); - drop(queued); // Accepted job still owns the only waiting slot. - let rejected = observed( - fixture - .inner_router() - .oneshot(Fixture::subject_request("rejected-subject")), - ) - .await - .unwrap(); - let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await; - let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap(); - assert!( - error.message.contains("capacity"), - "typed overload error: {body}" - ); - assert_eq!(subject_count(&path, "queued-subject"), 0); - assert_eq!(subject_count(&path, "rejected-subject"), 0); - let mut drained = Box::pin(fixture.executor.drain()); - assert!(futures::poll!(&mut drained).is_pending()); - lock.release(); - busy.resume(); - response_json(observed(running).await.unwrap(), StatusCode::CREATED).await; - observed(drained).await.unwrap(); - assert_eq!( - subject_count(&path, "queued-subject"), - 1, - "cancelled accepted mutation persists exactly once" - ); - assert_eq!( - subject_count(&path, "rejected-subject"), - 0, - "overload must not execute rejected SQL" - ); - let retry = fixture.dispatch(Fixture::subject_request("after-drain")); - response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await; - fixture.executor.drain().await.unwrap(); - assert_eq!( - subject_count(&path, "after-drain"), - 1, - "drain must release actual-job capacity" - ); -} - -#[tokio::test(start_paused = true)] -async fn saturated_cancelled_host_jobs_keep_their_budget_but_not_other_fixture_or_cleanup_admission() - { - use std::sync::{ - Condvar, - atomic::{AtomicUsize, Ordering}, - }; - // A drop guard releases workers even when an assertion panics. - struct Release(Arc<(Mutex, Condvar)>); - impl Drop for Release { - fn drop(&mut self) { - *self.0.0.lock().unwrap() = true; - self.0.1.notify_all(); - } - } - let owner = HostExecutors::default(); - let other = HostExecutors::default(); - let host = owner.host().unwrap(); - let gate = Arc::new((Mutex::new(false), Condvar::new())); - let release = Release(gate.clone()); - let completed = Arc::new(AtomicUsize::new(0)); - let mut waiters = Vec::new(); - for _ in 0..2 { - let host = host.clone(); - let gate = gate.clone(); - let completed = completed.clone(); - let (started, accepted) = tokio::sync::oneshot::channel(); - waiters.push(tokio::spawn(async move { - host.run(move || { - started.send(()).unwrap(); - let mut released = gate.0.lock().unwrap(); - while !*released { - released = gate.1.wait(released).unwrap(); - } - completed.fetch_add(1, Ordering::SeqCst); - Ok::<_, Error>(()) - }) - .await - })); - accepted.await.unwrap(); - } - for _ in 0..16 { - let completed = completed.clone(); - host.dispatch(move || { - completed.fetch_add(1, Ordering::SeqCst); - }) - .unwrap(); - } - for waiter in waiters { - waiter.abort(); - assert!(waiter.await.unwrap_err().is_cancelled()); - } - assert!(matches!( - host.run(|| Ok::<_, Error>(())).await, - Err(Error::DbCapacityExceeded) - )); - // Paused Tokio time can advance without completing or releasing actual Host - // work. Neither another fixture nor cleanup needs this saturated Host lane. - tokio::time::advance(std::time::Duration::from_secs(31)).await; - assert_eq!(completed.load(Ordering::SeqCst), 0); - other - .host() - .unwrap() - .run(|| Ok::<_, Error>(())) - .await - .unwrap(); - owner - .cleanup() - .unwrap() - .run(|| Ok::<_, Error>(())) - .await - .unwrap(); - assert!(matches!( - host.run(|| Ok::<_, Error>(())).await, - Err(Error::DbCapacityExceeded) - )); - drop(release); - host.drain().await.unwrap(); - assert_eq!(completed.load(Ordering::SeqCst), 18); - host.run(|| Ok::<_, Error>(())).await.unwrap(); - host.drain().await.unwrap(); -} - -#[tokio::test(flavor = "current_thread")] -async fn restore_intent_pins_once_after_write_ownership_without_a_deferred_read_upgrade() { - let fixture = Fixture::new().await; - let store = fixture.api.config_store.clone(); - let (mut busy, waiting) = BusyObservation::install_store(store.clone()); - let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); - let worker = RuntimeWorkerRef::new("restore-runtime", "restore-worker"); - let pin_store = store.clone(); - let pin_worker = worker.clone(); - let mut pinned = Box::pin(fixture.executor.run(move || { - pin_store.pin_internal_worker_restore_intent( - WORKSPACE, - &pin_worker, - "restore-domain", - Some("observed-before-dispatch"), - Some("restore-request"), - ) - })); - tokio::select! { - result = &mut pinned => panic!("restore intent must wait for write ownership before reading, not fail a deferred read upgrade: {result:?}"), - arrived = observed(waiting) => arrived.unwrap(), - } - runtime_progress_while_locked(&lock).await; - assert!(futures::poll!(&mut pinned).is_pending()); - lock.release(); - busy.resume(); - let request = observed(pinned).await.unwrap(); - assert_eq!(request.request_id, "restore-request"); - assert_eq!( - request.expected_observation_token, - "observed-before-dispatch" - ); - fixture.executor.drain().await.unwrap(); - let saved = store - .get_internal_worker_restore_intent(WORKSPACE, &worker, &request.request_id) - .unwrap() - .unwrap(); - assert_eq!(saved.request_id, request.request_id); - assert_eq!( - saved.expected_observation_token, - request.expected_observation_token - ); - let count: i64 = store - .with_conn(|conn| { - Ok(conn.query_row( - "SELECT count(*) FROM worker_restore_intents WHERE workspace_id=?1 AND request_id=?2", - rusqlite::params![WORKSPACE, request.request_id], |row| row.get(0) - )?) - }) - .unwrap(); - assert_eq!( - count, 1, - "one admitted intent must persist exactly once, with no operation retry" - ); -} diff --git a/crates/workspace-server/src/server_drive.rs b/crates/workspace-server/src/server_drive.rs index 33bb2e908..57c79ef7f 100644 --- a/crates/workspace-server/src/server_drive.rs +++ b/crates/workspace-server/src/server_drive.rs @@ -51,13 +51,14 @@ async fn blocking( mutation: bool, action: impl FnOnce() -> workspace_drive::Result + Send + 'static, ) -> DriveResult { - crate::db_executor::DbExecutor::shared() - .run(move || Ok::<_, Error>(action())) + tokio::task::spawn_blocking(action) .await - .map_err(|error| match error { - Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503), - _ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown), - _ => failure(503), + .map_err(|_| { + if mutation { + DriveApiError::new(DriveApiErrorCode::OutcomeUnknown) + } else { + failure(503) + } })? .map_err(|error| domain_error(error, mutation)) } diff --git a/crates/workspace-server/src/server_drive_grants.rs b/crates/workspace-server/src/server_drive_grants.rs index 4d1f82fe5..9d85eaf67 100644 --- a/crates/workspace-server/src/server_drive_grants.rs +++ b/crates/workspace-server/src/server_drive_grants.rs @@ -24,10 +24,7 @@ pub(super) fn pre_error(error: impl Into) -> DriveApiError { }) } fn mutation_error(error: Error) -> DriveApiError { - if matches!( - error, - Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown - ) { + if matches!(error, Error::Sqlite(_) | Error::Store(_)) { fail(DriveApiErrorCode::OutcomeUnknown) } else { pre_error(error) @@ -100,27 +97,23 @@ pub(super) async fn authorized_drive( }; // Reject missing/read-only grants before upload body reception. This check // is admission only: every later operation rechecks under its own lock. - api.store - .db_executor() - .run(move || { - Ok::<_, Error>((|| { - drive.check_authorized(write)?; - Ok::<_, workspace_drive::Error>(drive) - })()) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? - .map_err(|error| { - fail(match error { - workspace_drive::Error::Denied => DriveApiErrorCode::Denied, - workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { - DriveApiErrorCode::Conflict - } - workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, - workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, - _ => DriveApiErrorCode::StorageUnavailable, - }) + tokio::task::spawn_blocking(move || { + drive.check_authorized(write)?; + Ok::<_, workspace_drive::Error>(drive) + }) + .await + .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? + .map_err(|error| { + fail(match error { + workspace_drive::Error::Denied => DriveApiErrorCode::Denied, + workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { + DriveApiErrorCode::Conflict + } + workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, + workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, + _ => DriveApiErrorCode::StorageUnavailable, }) + }) } pub(super) async fn create_grant( @@ -159,10 +152,9 @@ pub(super) async fn create_grant( revoked_at: None, }; let store = api.store.clone(); - api.store - .db_executor() - .run(move || store.create_workspace_drive_grant(&grant)) + tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant)) .await + .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? .map_err(mutation_error) } @@ -183,12 +175,13 @@ pub(super) async fn revoke_grant( actor.account_id.clone(), ); let store = api.store.clone(); - api.store - .db_executor() - .run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor)) - .await - .map_err(mutation_error)? - .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) + tokio::task::spawn_blocking(move || { + store.revoke_workspace_drive_grant(&workspace, &grant, &actor) + }) + .await + .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? + .map_err(mutation_error)? + .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) } fn validate_grant_id(id: &str) -> DriveResult<()> { @@ -219,14 +212,12 @@ pub(super) async fn list_grants( } let workspace = workspace_id.to_owned(); let store = api.store.clone(); - let mut grants = api - .store - .db_executor() - .run(move || { - store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) - }) - .await - .map_err(pre_error)?; + let mut grants = tokio::task::spawn_blocking(move || { + store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) + }) + .await + .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? + .map_err(pre_error)?; let more = grants.len() > limit; grants.truncate(limit); let next_after = if more { diff --git a/crates/workspace-server/src/server_ticket_case_signed_tests.rs b/crates/workspace-server/src/server_ticket_case_signed_tests.rs index 8f7c38b52..78ba317b2 100644 --- a/crates/workspace-server/src/server_ticket_case_signed_tests.rs +++ b/crates/workspace-server/src/server_ticket_case_signed_tests.rs @@ -724,7 +724,7 @@ async fn catalog_case_dispatch_authenticates_and_routes_the_canonical_workspace_ reason: ticket::TicketClosureReason::Completed, }, ); - let catalog = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone()); + let catalog = WorkspaceServerApi::new(api.config.clone(), api.store.clone()); // Retain only this in-process fixture: do not create a real Runtime or a // background Orchestrator hook while testing catalog authentication/routing. catalog diff --git a/crates/workspace-server/src/server_ticket_tags_tests.rs b/crates/workspace-server/src/server_ticket_tags_tests.rs index 1608982bd..b636a8356 100644 --- a/crates/workspace-server/src/server_ticket_tags_tests.rs +++ b/crates/workspace-server/src/server_ticket_tags_tests.rs @@ -529,7 +529,7 @@ async fn runtime_tag_mutations_reject_invalid_proofs_and_changed_binding_authori *tampered.body_mut() = Body::from(format!("{} ", input)); runtime_tag_response(app.clone(), tampered, StatusCode::UNAUTHORIZED).await; } - let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); + let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); let original = api .store .get_workspace_runtime_binding(TEST_WORKSPACE_ID, "runtime-tags") diff --git a/crates/workspace-server/src/server_workspace_config.rs b/crates/workspace-server/src/server_workspace_config.rs index f1372b1e4..0e0a54959 100644 --- a/crates/workspace-server/src/server_workspace_config.rs +++ b/crates/workspace-server/src/server_workspace_config.rs @@ -47,10 +47,7 @@ pub(super) fn pre_error(error: impl Into) -> Failure { fn mutation_error(error: Error) -> Failure { // Storage/service failures at a mutation boundary may include a commit // acknowledgement failure. Never describe those as a guaranteed rollback. - if matches!( - &error, - Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown - ) { + if matches!(&error, Error::Sqlite(_) | Error::Store(_)) { unknown() } else { pre_error(error) @@ -189,23 +186,9 @@ pub(super) async fn get( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> ConfigResult> { - let guard = lock(api, worker).await?; - let executor = api.store.db_executor(); - let api = api.clone(); - let worker = worker.clone(); - executor - .run(move || { - let _guard = guard; - Ok::<_, Error>((|| { - let grant = grant_for_worker(&api, &worker)?; - Ok::<_, Failure>( - active_link(&api, &worker, &grant, None)? - .map(|link| attachment(&grant, &link, true)), - ) - })()) - }) - .await - .map_err(pre_error)? + let _guard = lock(api, worker).await?; + let grant = grant_for_worker(api, worker)?; + Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true))) } pub(super) async fn attach( api: &WorkspaceApi, @@ -215,59 +198,46 @@ pub(super) async fn attach( if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) { return Err(fail(400, "invalid_alias")); } - let guard = lock(api, worker).await?; - let executor = api.store.db_executor(); - let api = api.clone(); - let worker = worker.clone(); - executor - .run(move || { - let _guard = guard; - Ok::<_, Error>((|| { - let grant = grant_for_worker(&api, &worker)?; - let access = request.access.unwrap_or(grant.access); - if access == Access::ReadWrite && grant.access != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - if let Some(mut link) = active_link(&api, &worker, &grant, None)? { - // Same active alias/identity is idempotent; explicit write never silently downgrades. - if request.access == Some(Access::ReadWrite) - && effective_access(&grant, &link) != Access::ReadWrite - { - return Err(fail(403, "access_denied")); - } - if access == Access::ReadOnly - && effective_access(&grant, &link) != Access::ReadOnly - { - link.capabilities = access_capabilities(access); - link = api - .store - .attach_worker_workdir(&link) - .map_err(mutation_error)?; - } - return Ok(attachment(&grant, &link, true)); - } - let link = api - .store - .attach_worker_workdir(&WorkerWorkdirLinkRecord { - connection_id: String::new(), - workspace_id: grant.workspace_id.clone(), - worker: worker.clone(), - workdir_id: grant.working_directory_id.clone(), - alias: ALIAS.into(), - capabilities: access_capabilities(access), - linked_at: now_registry_timestamp(), - unlinked_at: None, - }) - .map_err(mutation_error)?; - // No Runtime filesystem/session synchronization for a logical config target. - api.worker_projection - .refresh(&worker) - .map_err(|_| unknown())?; - Ok(attachment(&grant, &link, false)) - })()) + let _guard = lock(api, worker).await?; + let grant = grant_for_worker(api, worker)?; + let access = request.access.unwrap_or(grant.access); + if access == Access::ReadWrite && grant.access != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + if let Some(mut link) = active_link(api, worker, &grant, None)? { + // Same active alias/identity is idempotent; explicit write never silently downgrades. + if request.access == Some(Access::ReadWrite) + && effective_access(&grant, &link) != Access::ReadWrite + { + return Err(fail(403, "access_denied")); + } + if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly { + link.capabilities = access_capabilities(access); + link = api + .store + .attach_worker_workdir(&link) + .map_err(mutation_error)?; + } + return Ok(attachment(&grant, &link, true)); + } + let link = api + .store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: grant.workspace_id.clone(), + worker: worker.clone(), + workdir_id: grant.working_directory_id.clone(), + alias: ALIAS.into(), + capabilities: access_capabilities(access), + linked_at: now_registry_timestamp(), + unlinked_at: None, }) - .await - .map_err(mutation_error)? + .map_err(mutation_error)?; + // No Runtime filesystem/session synchronization for a logical config target. + api.worker_projection + .refresh(worker) + .map_err(|_| unknown())?; + Ok(attachment(&grant, &link, false)) } fn resolve( api: &WorkspaceApi, @@ -343,22 +313,9 @@ pub(super) async fn observe( for path in &request.paths { path_valid(path, true)?; } - let guard = lock(api, worker).await?; - let resolution_api = api.clone(); - let resolution_worker = worker.clone(); - let connection_id = request.connection_id.clone(); - let (grant, link, state, _guard) = api - .store - .db_executor() - .run(move || { - Ok::<_, Error>((|| { - let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; - let state = tree(&resolution_api)?; - Ok::<_, Failure>((grant, link, state, guard)) - })()) - }) - .await - .map_err(pre_error)??; + let _guard = lock(api, worker).await?; + let (grant, link) = resolve(api, worker, &request.connection_id)?; + let state = tree(api)?; let schema_fingerprint = api .config_schema_registry .compose() @@ -470,22 +427,9 @@ pub(super) async fn read( request: WorkspaceConfigReadRequest, ) -> ConfigResult { path_valid(&request.path, false)?; - let guard = lock(api, worker).await?; - let resolution_api = api.clone(); - let resolution_worker = worker.clone(); - let connection_id = request.connection_id.clone(); - let (grant, link, state, _guard) = api - .store - .db_executor() - .run(move || { - Ok::<_, Error>((|| { - let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; - let state = tree(&resolution_api)?; - Ok::<_, Failure>((grant, link, state, guard)) - })()) - }) - .await - .map_err(pre_error)??; + let _guard = lock(api, worker).await?; + let (grant, link) = resolve(api, worker, &request.connection_id)?; + let state = tree(api)?; let schema_fingerprint = api .config_schema_registry .compose() @@ -538,25 +482,12 @@ pub(super) async fn commit( server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?, } } - let guard = lock(api, worker).await?; - let resolution_api = api.clone(); - let resolution_worker = worker.clone(); - let connection_id = request.connection_id.clone(); - let (grant, link, state, guard) = api - .store - .db_executor() - .run(move || { - Ok::<_, Error>((|| { - let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; - if effective_access(&grant, &link) != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - let state = tree(&resolution_api)?; - Ok::<_, Failure>((grant, link, state, guard)) - })()) - }) - .await - .map_err(pre_error)??; + let _guard = lock(api, worker).await?; + let (grant, link) = resolve(api, worker, &request.connection_id)?; + if effective_access(&grant, &link) != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + let state = tree(api)?; let schema_fingerprint = api .config_schema_registry .compose() @@ -570,19 +501,17 @@ pub(super) async fn commit( // Evaluation/semantic validation are pure preparation. Bound this stage // without ever dispatching persistence in a timed-out background task. let preparation_api = api.clone(); - let executor = api.store.db_executor(); - let preparation = executor.run(move || { - // Timeout/cancellation releases exclusion only after actual preparation ends. - let candidate = prepare_workspace_config_tree( + let preparation = tokio::task::spawn_blocking(move || { + prepare_workspace_config_tree( &preparation_api, &preparation_api.config.workspace_id, &canonical, - )?; - Ok::<_, Error>((candidate, guard)) + ) }); - let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation) + let candidate = tokio::time::timeout(DEADLINE, preparation) .await .map_err(|_| fail(503, "evaluation_timeout"))? + .map_err(|_| fail(500, "evaluation_unavailable"))? .map_err(pre_error)?; if candidate.base_digest != state.snapshot.digest || candidate.base_toolchain_fingerprint != state.contract.fingerprint @@ -591,78 +520,15 @@ pub(super) async fn commit( { return Err(fail(409, "stale_validator")); } - let persistence_api = api.clone(); - let worker = worker.clone(); - let connection_id = request.connection_id.clone(); - let expected_grant = grant.clone(); - let expected_link = link.clone(); - let (state, publication) = api - .store - .db_executor() - .run(move || { - Ok::<_, Error>((|| { - let (current_grant, current_link) = - resolve(&persistence_api, &worker, &connection_id)?; - if current_grant != expected_grant - || current_link.capabilities != expected_link.capabilities - { - return Err(fail(409, "authority_changed")); - } - // Preserve the shared helper's ordering and atomic CAS, but never - // run its remote Runtime publication on the sole DB worker. - project_repository_access_candidate( - &*persistence_api.store, - &persistence_api.repository_secrets, - &persistence_api.config.workspace_id, - &candidate, - ) - .map_err(|_| unknown())?; - let state = persistence_api - .config_store - .commit_evaluated_workspace_config( - &persistence_api.config.workspace_id, - &candidate, - ) - .map_err(|_| unknown())?; - let publication = persistence_api - .prompt_projection_cache - .resolve(&persistence_api.config.workspace_id, &state) - .ok() - .and_then(|projection| { - // Dispatch from the actual persistence owner even if its - // waiter was cancelled. Runtime I/O owns exclusion, not - // a DB worker or an open SQL transaction. - let (complete, publication) = tokio::sync::oneshot::channel(); - let runtime = persistence_api.runtime.clone(); - let admission = persistence_api.host_executors.host().and_then(|executor| { - executor.dispatch(move || { - let _guard = guard; - let _diagnostics = runtime - .observe_workspace_prompt_projection((*projection).clone()); - let _ = complete.send(()); - }) - }); - match admission { - Ok(()) => Some(publication), - Err(error) => { - tracing::warn!( - workspace_id = %persistence_api.config.workspace_id, - %error, - "Workspace config committed but Runtime publication was not admitted" - ); - None - } - } - }); - Ok((state, publication)) - })()) - }) - .await - .map_err(mutation_error)??; - if let Some(publication) = publication { - // Publication is best effort and cannot turn an acknowledged CAS into an error. - let _ = publication.await; + // Recheck durable authority before the first auxiliary/canonical effects. + let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?; + if current_grant != grant || current_link.capabilities != link.capabilities { + return Err(fail(409, "authority_changed")); } + // From this boundary repository credential projection may have side effects. + // Never falsely claim rollback or automatically replay a failed/uncertain save. + let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate) + .map_err(|_| unknown())?; Ok(WorkspaceConfigCommitResponse { validator: validator(&grant, &link, &state, &schema_fingerprint, ""), digest: state.snapshot.digest, @@ -678,61 +544,37 @@ pub(super) async fn create_grant( .await .map_err(pre_error)?; let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id); - let guard = lock(api, &worker).await?; - let runtime = api.runtime.clone(); - let lookup_worker = worker.clone(); - let (summary, guard) = api - .host_executors - .host() - .map_err(pre_error)? - .run(move || { - Ok::<_, Error>((|| { - let summary = runtime - .worker(&lookup_worker) - .map_err(|_| fail(404, "worker_not_found"))?; - Ok::<_, Failure>((summary, guard)) - })()) - }) - .await - .map_err(pre_error)??; + let _guard = lock(api, &worker).await?; + let summary = api + .runtime + .worker(&worker) + .map_err(|_| fail(404, "worker_not_found"))?; if summary.workspace.workspace_id.as_deref() != Some(workspace_id) { return Err(fail(403, "access_denied")); } - let executor = api.store.db_executor(); - let api = api.clone(); - let workspace_id = workspace_id.to_owned(); - let account_id = actor.account_id.clone(); - executor - .run(move || { - let _guard = guard; - Ok::<_, Error>((|| { - if let Some(grant) = api - .store - .current_workspace_config_grant(&workspace_id, &worker) - .map_err(pre_error)? - { - if grant.access != request.access { - return Err(fail(409, "grant_exists_revoke_first")); - } - return Ok(grant); - } - let grant = WorkspaceConfigGrantResponse { - grant_id: Uuid::now_v7().to_string(), - workspace_id, - runtime_id: worker.runtime_id, - worker_id: worker.worker_id, - working_directory_id: Uuid::now_v7().to_string(), - access: request.access, - revoked: false, - }; - api.store - .create_workspace_config_grant(&grant, &account_id) - .map_err(mutation_error)?; - Ok(grant) - })()) - }) - .await - .map_err(mutation_error)? + if let Some(grant) = api + .store + .current_workspace_config_grant(workspace_id, &worker) + .map_err(pre_error)? + { + if grant.access != request.access { + return Err(fail(409, "grant_exists_revoke_first")); + } + return Ok(grant); + } + let grant = WorkspaceConfigGrantResponse { + grant_id: Uuid::now_v7().to_string(), + workspace_id: workspace_id.into(), + runtime_id: worker.runtime_id, + worker_id: worker.worker_id, + working_directory_id: Uuid::now_v7().to_string(), + access: request.access, + revoked: false, + }; + api.store + .create_workspace_config_grant(&grant, &actor.account_id) + .map_err(mutation_error)?; + Ok(grant) } pub(super) async fn revoke_grant( api: &WorkspaceApi, @@ -743,60 +585,39 @@ pub(super) async fn revoke_grant( require_workspace_owner(api, workspace_id, actor, "Workspace config grants") .await .map_err(pre_error)?; - let lookup_api = api.clone(); - let lookup_workspace_id = workspace_id.to_owned(); - let lookup_grant_id = grant_id.to_owned(); let mut grant = api .store - .db_executor() - .run(move || { - lookup_api - .store - .get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id) - }) - .await + .get_workspace_config_grant(workspace_id, grant_id) .map_err(pre_error)? .ok_or_else(|| fail(404, "not_found"))?; let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id); - let guard = lock(api, &worker).await?; - let executor = api.store.db_executor(); - let api = api.clone(); - let workspace_id = workspace_id.to_owned(); - let grant_id = grant_id.to_owned(); - executor - .run(move || { - let _guard = guard; - Ok::<_, Error>((|| { - api.store - .revoke_workspace_config_grant(&workspace_id, &grant_id) - .map_err(mutation_error)?; - grant.revoked = true; - // Revocation is durable before unlink; a failed cleanup cannot restore access. - if let Some(link) = api - .store - .list_workdir_worker_links(&workspace_id, &grant.working_directory_id) - .map_err(|_| unknown())? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.worker == worker) - { - api.store - .detach_worker_workdir_connection( - &workspace_id, - &worker, - &link.alias, - &link.connection_id, - &now_registry_timestamp(), - ) - .map_err(|_| unknown())?; - } - api.worker_projection - .refresh(&worker) - .map_err(|_| unknown())?; - Ok(grant) - })()) - }) - .await - .map_err(mutation_error)? + let _guard = lock(api, &worker).await?; + api.store + .revoke_workspace_config_grant(workspace_id, grant_id) + .map_err(mutation_error)?; + grant.revoked = true; + // Revocation is durable before unlink; a failed cleanup cannot restore access. + if let Some(link) = api + .store + .list_workdir_worker_links(workspace_id, &grant.working_directory_id) + .map_err(|_| unknown())? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.worker == worker) + { + api.store + .detach_worker_workdir_connection( + workspace_id, + &worker, + &link.alias, + &link.connection_id, + &now_registry_timestamp(), + ) + .map_err(|_| unknown())?; + } + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(grant) } pub(super) fn summary( @@ -834,11 +655,7 @@ pub(super) fn summary( mod tests { use super::*; #[test] - fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() { - assert_eq!( - mutation_error(Error::DbOperationOutcomeUnknown).classification, - Classification::Unknown - ); + fn workspace_config_mutation_storage_failures_are_conservatively_unknown() { assert_eq!( mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification, Classification::Unknown diff --git a/crates/workspace-server/src/server_workspace_config_tests.rs b/crates/workspace-server/src/server_workspace_config_tests.rs index 1c932be1a..ac11e7766 100644 --- a/crates/workspace-server/src/server_workspace_config_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_tests.rs @@ -318,7 +318,7 @@ mod workspace_config_integration { }; for scoped in [false, true] { let app = if scoped { - workspace_server_router(WorkspaceServerApi::new_for_test( + workspace_server_router(WorkspaceServerApi::new( api.config.clone(), api.store.clone(), )) @@ -803,9 +803,9 @@ mod workspace_config_integration { .supports(workdir::WorkdirSessionCapability::Command) ); let lock = current_worker_session_lock(api, &fixture.worker); - let _guard = Arc::new(lock.lock_owned().await); + let _guard = lock.lock().await; assert!( - open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard) + open_current_worker_workdir_session_locked(api, &fixture.worker, &link) .await .is_err() ); @@ -1385,7 +1385,7 @@ mod workspace_config_integration { let api = &fixture.api; let granted = grant(api, &fixture.worker, Access::ReadWrite).await; let attached = attach(api, &fixture.worker).await; - let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap(); + let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); assert_eq!( restarted .get_workspace_config_grant(TEST_WORKSPACE_ID, &granted.grant_id) diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 1e989edd8..b5a229f55 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -800,7 +800,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w let base = format!("http://{addr}"); let mut config = test_server_config(root.path()); config.backend_base_url = Some("server-test".into()); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); let captured = Arc::new(Mutex::new(None)); let abort = Arc::new(Mutex::new(None)); let factory = ConfigCaptureFactory { diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 4585493d5..3e44c9081 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -1131,10 +1131,6 @@ impl WorkspaceResourceKind { #[async_trait] pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { - fn db_executor(&self) -> crate::db_executor::DbExecutor { - crate::db_executor::DbExecutor::shared() - } - async fn schema_version(&self) -> Result; fn resource_key( &self, @@ -2261,7 +2257,6 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { #[derive(Clone)] pub struct SqliteWorkspaceStore { conn: Arc>, - db_executor: crate::db_executor::DbExecutor, } impl SqliteWorkspaceStore { @@ -2302,50 +2297,12 @@ impl SqliteWorkspaceStore { } pub fn from_connection(conn: Connection) -> Result { - Self::from_connection_with_executor(conn, crate::db_executor::DbExecutor::shared()) - } - - fn from_connection_with_executor( - conn: Connection, - db_executor: crate::db_executor::DbExecutor, - ) -> Result { prepare_connection(&conn)?; Ok(Self { conn: Arc::new(Mutex::new(conn)), - db_executor, }) } - // Ordinary fixtures own the same finite budget as production, not another - // fixture's admission/SQL/drain lifetime or a permanent global worker. - #[cfg(test)] - pub(crate) fn open_for_test(path: impl AsRef) -> Result { - Self::from_connection_for_test(Connection::open(path)?) - } - - #[cfg(test)] - pub(crate) fn in_memory_for_test() -> Result { - Self::from_connection_for_test(Connection::open_in_memory()?) - } - - #[cfg(test)] - pub(crate) fn from_connection_for_test(conn: Connection) -> Result { - Self::from_connection_with_executor( - conn, - crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits::default())?, - ) - } - - /// Override execution limits for a hermetic fixture; subsequent clones share it. - pub fn with_db_executor(mut self, executor: crate::db_executor::DbExecutor) -> Self { - self.db_executor = executor; - self - } - - pub fn db_executor(&self) -> crate::db_executor::DbExecutor { - self.db_executor.clone() - } - pub(crate) fn with_conn(&self, f: impl FnOnce(&Connection) -> Result) -> Result { let conn = self .conn @@ -2365,16 +2322,12 @@ impl SqliteWorkspaceStore { f(&mut conn) } - pub(crate) fn with_immediate_transaction( + pub(crate) fn with_transaction( &self, f: impl FnOnce(&rusqlite::Transaction<'_>) -> Result, ) -> Result { self.with_conn_mut(|conn| { - // These callers read authority/journal state before mutating it. - // Acquire write ownership before the first read: a DEFERRED read - // upgrade against another connection's writer can return BUSY - // without invoking the busy handler. Never retry the domain mutation. - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let tx = conn.transaction()?; let value = f(&tx)?; tx.commit()?; Ok(value) @@ -2435,7 +2388,7 @@ impl SqliteWorkspaceStore { observed_token: Option<&str>, explicit_request_id: Option<&str>, ) -> Result { - self.with_immediate_transaction(|tx| { + self.with_transaction(|tx| { let existing: Option<(String, String)> = if let Some(id) = explicit_request_id { tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4", params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()? @@ -3919,16 +3872,8 @@ fn insert_workspace_runtime_binding_audit( #[async_trait] impl ControlPlaneStore for SqliteWorkspaceStore { - fn db_executor(&self) -> crate::db_executor::DbExecutor { - self.db_executor.clone() - } - async fn schema_version(&self) -> Result { - let store = self.clone(); - - self.db_executor - .run(move || store.with_conn(current_schema_version)) - .await + self.with_conn(current_schema_version) } fn resource_key( @@ -4060,114 +4005,100 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()> { - let store = self.clone(); - let record = record.clone(); - self.db_executor - .run(move || { - let store = &store; - let record = &record; - store.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - #[cfg(test)] - if record.owner_account_id == "owner-account" { - // Unrelated store tests use this explicit fixture identity. Keep the - // production owner contract strict while giving those fixtures a real - // User Account row instead of reviving ownerless Workspace setup. - tx.execute( - "INSERT OR IGNORE INTO accounts ( - account_id, kind, handle, display_name, created_at, updated_at - ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", - params![record.owner_account_id.as_str(), record.created_at.as_str()], - )?; - } - let owner_kind = tx - .query_row( - "SELECT kind FROM accounts WHERE account_id = ?1", - params![record.owner_account_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if owner_kind.as_deref() != Some("user") { - return Err(Error::Store(format!( - "Workspace owner `{}` must reference an existing User Account", - record.owner_account_id - ))); - } - let current_owner = tx - .query_row( - "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", - params![record.workspace_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if current_owner - .as_deref() - .is_some_and(|owner| owner != record.owner_account_id) - { - return Err(Error::Store(format!( - "Workspace `{}` owner is immutable through upsert", - record.workspace_id - ))); - } - tx.execute( - r#"INSERT INTO workspaces ( - workspace_id, owner_account_id, display_name, state, created_at, updated_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(workspace_id) DO UPDATE SET - display_name = excluded.display_name, - state = excluded.state, - updated_at = excluded.updated_at"#, - params![ - record.workspace_id, - record.owner_account_id, - record.display_name, - record.state, - record.created_at, - record.updated_at, - ], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_memory_settings ( - workspace_id, language, created_at, updated_at - ) VALUES (?1, 'English', ?2, ?3)"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_signing_identities ( - workspace_id, key_id, algorithm, public_key, public_key_fingerprint, - private_material_ref, state, created_at, provisioned_at, updated_at - ) VALUES ( - ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, - 'workspace-signing/' || ?1 || '/ed25519-v1', - 'pending_provisioning', ?2, NULL, ?3 - )"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.commit()?; - Ok(()) - })?; - store.materialize_workspace_config(&record.workspace_id, &record.created_at) - }) - .await + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + #[cfg(test)] + if record.owner_account_id == "owner-account" { + // Unrelated store tests use this explicit fixture identity. Keep the + // production owner contract strict while giving those fixtures a real + // User Account row instead of reviving ownerless Workspace setup. + tx.execute( + "INSERT OR IGNORE INTO accounts ( + account_id, kind, handle, display_name, created_at, updated_at + ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", + params![record.owner_account_id.as_str(), record.created_at.as_str()], + )?; + } + let owner_kind = tx + .query_row( + "SELECT kind FROM accounts WHERE account_id = ?1", + params![record.owner_account_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if owner_kind.as_deref() != Some("user") { + return Err(Error::Store(format!( + "Workspace owner `{}` must reference an existing User Account", + record.owner_account_id + ))); + } + let current_owner = tx + .query_row( + "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", + params![record.workspace_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if current_owner + .as_deref() + .is_some_and(|owner| owner != record.owner_account_id) + { + return Err(Error::Store(format!( + "Workspace `{}` owner is immutable through upsert", + record.workspace_id + ))); + } + tx.execute( + r#"INSERT INTO workspaces ( + workspace_id, owner_account_id, display_name, state, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(workspace_id) DO UPDATE SET + display_name = excluded.display_name, + state = excluded.state, + updated_at = excluded.updated_at"#, + params![ + record.workspace_id, + record.owner_account_id, + record.display_name, + record.state, + record.created_at, + record.updated_at, + ], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_memory_settings ( + workspace_id, language, created_at, updated_at + ) VALUES (?1, 'English', ?2, ?3)"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_signing_identities ( + workspace_id, key_id, algorithm, public_key, public_key_fingerprint, + private_material_ref, state, created_at, provisioned_at, updated_at + ) VALUES ( + ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, + 'workspace-signing/' || ?1 || '/ed25519-v1', + 'pending_provisioning', ?2, NULL, ?3 + )"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.commit()?; + Ok(()) + })?; + self.materialize_workspace_config(&record.workspace_id, &record.created_at) } async fn get_workspace(&self, workspace_id: &str) -> Result> { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - self.db_executor.run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - store.with_conn(|conn| { - conn.query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional() - .map_err(Error::from) - }) - }).await + self.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional() + .map_err(Error::from) + }) } async fn update_workspace_display_name( @@ -4176,68 +4107,58 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_updated_at: &str, display_name: &str, ) -> Result> { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let expected_updated_at = expected_updated_at.to_owned(); - let display_name = display_name.to_owned(); - self.db_executor.run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let expected_updated_at = expected_updated_at.as_str(); - let display_name = display_name.as_str(); - validate_identifier("workspace_id", workspace_id)?; - validate_non_empty("expected_updated_at", expected_updated_at)?; - validate_non_empty("display_name", display_name)?; - store.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let current = tx - .query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional()?; - let Some(current) = current else { - tx.commit()?; - return Ok(None); - }; - if current.updated_at != expected_updated_at { - tx.commit()?; - return Ok(None); - } - if current.display_name == display_name { - tx.commit()?; - return Ok(Some(current)); - } - - let now = chrono::Utc::now(); - let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) - .ok() - .map(|previous| previous.with_timezone(&chrono::Utc)) - .filter(|previous| *previous >= now) - .map(|previous| previous + chrono::Duration::nanoseconds(1)) - .unwrap_or(now) - .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); - let changed = tx.execute( - r#"UPDATE workspaces - SET display_name = ?3, updated_at = ?4 - WHERE workspace_id = ?1 AND updated_at = ?2"#, - params![workspace_id, expected_updated_at, display_name, updated_at], - )?; - if changed != 1 { - tx.commit()?; - return Ok(None); - } - let updated = WorkspaceRecord { - display_name: display_name.to_string(), - updated_at, - ..current - }; + validate_identifier("workspace_id", workspace_id)?; + validate_non_empty("expected_updated_at", expected_updated_at)?; + validate_non_empty("display_name", display_name)?; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current = tx + .query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional()?; + let Some(current) = current else { tx.commit()?; - Ok(Some(updated)) - }) - }).await + return Ok(None); + }; + if current.updated_at != expected_updated_at { + tx.commit()?; + return Ok(None); + } + if current.display_name == display_name { + tx.commit()?; + return Ok(Some(current)); + } + + let now = chrono::Utc::now(); + let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) + .ok() + .map(|previous| previous.with_timezone(&chrono::Utc)) + .filter(|previous| *previous >= now) + .map(|previous| previous + chrono::Duration::nanoseconds(1)) + .unwrap_or(now) + .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + let changed = tx.execute( + r#"UPDATE workspaces + SET display_name = ?3, updated_at = ?4 + WHERE workspace_id = ?1 AND updated_at = ?2"#, + params![workspace_id, expected_updated_at, display_name, updated_at], + )?; + if changed != 1 { + tx.commit()?; + return Ok(None); + } + let updated = WorkspaceRecord { + display_name: display_name.to_string(), + updated_at, + ..current + }; + tx.commit()?; + Ok(Some(updated)) + }) } fn create_workspace_bootstrap( @@ -4757,17 +4678,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result> { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - SqliteWorkspaceStore::get_workspace_runtime_binding(store, workspace_id, runtime_id) - }) - .await + SqliteWorkspaceStore::get_workspace_runtime_binding(self, workspace_id, runtime_id) } async fn list_workspace_runtime_bindings( @@ -4775,19 +4686,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, include_revoked: bool, ) -> Result> { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - SqliteWorkspaceStore::list_workspace_runtime_bindings( - store, - workspace_id, - include_revoked, - ) - }) - .await + SqliteWorkspaceStore::list_workspace_runtime_bindings(self, workspace_id, include_revoked) } async fn has_other_active_workspace_runtime_binding( @@ -4795,21 +4694,11 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( - store, - workspace_id, - runtime_id, - ) - }) - .await + SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( + self, + workspace_id, + runtime_id, + ) } async fn delete_workspace_runtime_binding( @@ -4817,21 +4706,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - SqliteWorkspaceStore::delete_workspace_runtime_binding( - store, - workspace_id, - runtime_id, - ) - }) - .await + SqliteWorkspaceStore::delete_workspace_runtime_binding(self, workspace_id, runtime_id) } async fn reserve_runtime_removal( @@ -4843,48 +4718,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: &str, config_digest: &str, ) -> Result { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - let operation_id = operation_id.to_owned(); - let request_fingerprint = request_fingerprint.to_owned(); - let expected_binding_id = expected_binding_id.to_owned(); - let config_digest = config_digest.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - let operation_id = operation_id.as_str(); - let request_fingerprint = request_fingerprint.as_str(); - let expected_binding_id = expected_binding_id.as_str(); - let config_digest = config_digest.as_str(); - SqliteWorkspaceStore::reserve_runtime_removal( - store, - workspace_id, - runtime_id, - operation_id, - request_fingerprint, - expected_binding_id, - config_digest, - ) - }) - .await + SqliteWorkspaceStore::reserve_runtime_removal( + self, + workspace_id, + runtime_id, + operation_id, + request_fingerprint, + expected_binding_id, + config_digest, + ) } async fn get_runtime_removal( &self, operation_id: &str, ) -> Result> { - let store = self.clone(); - let operation_id = operation_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let operation_id = operation_id.as_str(); - SqliteWorkspaceStore::get_runtime_removal(store, operation_id) - }) - .await + SqliteWorkspaceStore::get_runtime_removal(self, operation_id) } async fn mark_runtime_removal_failed( @@ -4892,36 +4741,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, failure_category: &str, ) -> Result { - let store = self.clone(); - let operation_id = operation_id.to_owned(); - let failure_category = failure_category.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let operation_id = operation_id.as_str(); - let failure_category = failure_category.as_str(); - SqliteWorkspaceStore::mark_runtime_removal_failed( - store, - operation_id, - failure_category, - ) - }) - .await + SqliteWorkspaceStore::mark_runtime_removal_failed(self, operation_id, failure_category) } async fn commit_runtime_binding_removal( &self, operation_id: &str, ) -> Result { - let store = self.clone(); - let operation_id = operation_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let operation_id = operation_id.as_str(); - SqliteWorkspaceStore::commit_runtime_binding_removal(store, operation_id) - }) - .await + SqliteWorkspaceStore::commit_runtime_binding_removal(self, operation_id) } async fn complete_runtime_removal( @@ -4929,34 +4756,18 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, runtime_registration_removed: bool, ) -> Result { - let store = self.clone(); - let operation_id = operation_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let operation_id = operation_id.as_str(); - SqliteWorkspaceStore::complete_runtime_removal( - store, - operation_id, - runtime_registration_removed, - ) - }) - .await + SqliteWorkspaceStore::complete_runtime_removal( + self, + operation_id, + runtime_registration_removed, + ) } async fn list_resumable_runtime_removals( &self, workspace_id: &str, ) -> Result> { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - SqliteWorkspaceStore::list_resumable_runtime_removals(store, workspace_id) - }) - .await + SqliteWorkspaceStore::list_resumable_runtime_removals(self, workspace_id) } async fn upsert_workspace_runtime_binding_record( @@ -4964,14 +4775,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record: WorkspaceRuntimeBinding, replace: bool, ) -> Result { - let store = self.clone(); - - self.db_executor - .run(move || { - let store = &store; - SqliteWorkspaceStore::upsert_workspace_runtime_binding(store, record, replace) - }) - .await + SqliteWorkspaceStore::upsert_workspace_runtime_binding(self, record, replace) } async fn revoke_workspace_runtime_binding_record( @@ -4980,24 +4784,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, revoked_at: &str, ) -> Result { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - let revoked_at = revoked_at.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - let revoked_at = revoked_at.as_str(); - SqliteWorkspaceStore::revoke_workspace_runtime_binding( - store, - workspace_id, - runtime_id, - revoked_at, - ) - }) - .await + SqliteWorkspaceStore::revoke_workspace_runtime_binding( + self, + workspace_id, + runtime_id, + revoked_at, + ) } async fn put_workspace_runtime_binding_key( @@ -5006,22 +4798,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: Option<&str>, actor_account_id: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - let store = self.clone(); - let expected_binding_id = expected_binding_id.map(str::to_owned); - let actor_account_id = actor_account_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let expected_binding_id = expected_binding_id.as_deref(); - let actor_account_id = actor_account_id.as_str(); - SqliteWorkspaceStore::put_workspace_runtime_binding_key( - store, - record, - expected_binding_id, - actor_account_id, - ) - }) - .await + SqliteWorkspaceStore::put_workspace_runtime_binding_key( + self, + record, + expected_binding_id, + actor_account_id, + ) } async fn update_workspace_runtime_binding_metadata( @@ -5032,30 +4814,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { base_url: &str, updated_at: &str, ) -> Result { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - let display_name = display_name.to_owned(); - let base_url = base_url.to_owned(); - let updated_at = updated_at.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - let display_name = display_name.as_str(); - let base_url = base_url.as_str(); - let updated_at = updated_at.as_str(); - SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( - store, - workspace_id, - runtime_id, - display_name, - base_url, - updated_at, - ) - }) - .await + SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( + self, + workspace_id, + runtime_id, + display_name, + base_url, + updated_at, + ) } async fn revoke_workspace_runtime_binding_key( @@ -5066,30 +4832,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { actor_account_id: &str, revoked_at: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - let expected_binding_id = expected_binding_id.to_owned(); - let actor_account_id = actor_account_id.to_owned(); - let revoked_at = revoked_at.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - let expected_binding_id = expected_binding_id.as_str(); - let actor_account_id = actor_account_id.as_str(); - let revoked_at = revoked_at.as_str(); - SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( - store, - workspace_id, - runtime_id, - expected_binding_id, - actor_account_id, - revoked_at, - ) - }) - .await + SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( + self, + workspace_id, + runtime_id, + expected_binding_id, + actor_account_id, + revoked_at, + ) } async fn list_workspace_runtime_binding_audit( @@ -5098,22 +4848,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, limit: usize, ) -> Result> { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - SqliteWorkspaceStore::list_workspace_runtime_binding_audit( - store, - workspace_id, - runtime_id, - limit, - ) - }) - .await + SqliteWorkspaceStore::list_workspace_runtime_binding_audit( + self, + workspace_id, + runtime_id, + limit, + ) } async fn consume_worker_mutation_source_jti( @@ -5125,35 +4865,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore { now_seconds: u64, consumed_at: &str, ) -> Result { - let store = self.clone(); - let workspace_id = workspace_id.to_owned(); - let runtime_id = runtime_id.to_owned(); - let jti = jti.to_owned(); - let consumed_at = consumed_at.to_owned(); - self.db_executor - .run(move || { - let store = &store; - let workspace_id = workspace_id.as_str(); - let runtime_id = runtime_id.as_str(); - let jti = jti.as_str(); - let consumed_at = consumed_at.as_str(); - store.with_conn(|conn| { - let transaction = conn.unchecked_transaction()?; - transaction.execute( - "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", - params![now_seconds], - )?; - let inserted = transaction.execute( - r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( - workspace_id, runtime_id, jti, expires_at, consumed_at - ) VALUES (?1, ?2, ?3, ?4, ?5)"#, - params![workspace_id, runtime_id, jti, expires_at, consumed_at], - )?; - transaction.commit()?; - Ok(inserted == 1) - }) - }) - .await + self.with_conn(|conn| { + let transaction = conn.unchecked_transaction()?; + transaction.execute( + "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", + params![now_seconds], + )?; + let inserted = transaction.execute( + r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( + workspace_id, runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, ?2, ?3, ?4, ?5)"#, + params![workspace_id, runtime_id, jti, expires_at, consumed_at], + )?; + transaction.commit()?; + Ok(inserted == 1) + }) } fn plan_worker_removal( @@ -14960,7 +14686,7 @@ mod tests { fn drive_grants_migrate_and_preserve_audit_through_revoke_worker_delete_and_restart() { let t = tempfile::tempdir().unwrap(); let path = t.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let legacy = Connection::open_in_memory().unwrap(); configure_sqlite(&legacy).unwrap(); legacy.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY); @@ -15050,7 +14776,7 @@ mod tests { Some(other) ); drop(store); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); assert_eq!( store .get_workspace_drive_grant("drive-ws", &first.grant_id) @@ -15085,7 +14811,7 @@ mod tests { } #[test] fn drive_grants_baseline_constraints_and_worker_lifetime_fail_closed() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); let proposed = drive_grant_fixture(&store); let first = store.create_workspace_drive_grant(&proposed).unwrap(); store.with_conn(|c| { @@ -15121,7 +14847,7 @@ mod tests { fn schema_v74_migrates_historical_worker_projection_rows_without_counter_state() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("workspace.db"); - drop(SqliteWorkspaceStore::open_for_test(&path).unwrap()); + drop(SqliteWorkspaceStore::open(&path).unwrap()); let conn = Connection::open(&path).unwrap(); configure_sqlite(&conn).unwrap(); conn.execute_batch( @@ -15446,7 +15172,7 @@ mod tests { #[tokio::test] async fn external_workdir_grant_persists_typed_source_and_fences_generations() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store .upsert_account(&AccountRecord { account_id: "owner".to_string(), @@ -15818,7 +15544,7 @@ mod tests { fn runtime_removal_test_store() -> (tempfile::TempDir, SqliteWorkspaceStore) { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -16202,7 +15928,7 @@ mod tests { let path = temp.path().join("server.db"); prepare_retained_schema(&path, 59); - let migrated = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let migrated = SqliteWorkspaceStore::open(&path).unwrap(); migrated .with_conn(|conn| { let table_count: i64 = conn.query_row( @@ -16238,7 +15964,7 @@ mod tests { let path = temp.path().join("server.db"); prepare_retained_schema(&path, 60); - drop(SqliteWorkspaceStore::open_for_test(&path).unwrap()); + drop(SqliteWorkspaceStore::open(&path).unwrap()); let conn = Connection::open(&path).unwrap(); assert_eq!( current_schema_version(&conn).unwrap(), @@ -16268,7 +15994,7 @@ mod tests { #[test] fn schema_v59_runtime_removal_migration_rolls_back_partial_ddl_and_marker() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); store .with_conn(|conn| { let trigger_names = { @@ -16532,7 +16258,7 @@ mod tests { fn fresh_database_uses_only_the_canonical_workspace_baseline() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { let migrations = conn @@ -16680,7 +16406,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .collect::>(), (83..=LATEST_SCHEMA_VERSION).collect::>() ); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let assignment = store .get_active_ticket_worker_assignment("workspace-a", "ticket") .unwrap() @@ -17151,7 +16877,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let applied = SqliteWorkspaceStore::migrate_database(&path).unwrap(); assert_eq!(applied, plan); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -17700,7 +17426,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "migrated databases must reject incomplete singleton leases" ); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let legacy = RuntimeWorkerRef::new("legacy-runtime", "legacy-orchestrator"); let lease = store .require_current_worker_singleton_owner("workspace-a", &legacy) @@ -17888,7 +17614,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -17989,7 +17715,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(conn); - let error = SqliteWorkspaceStore::open_for_test(&path) + let error = SqliteWorkspaceStore::open(&path) .err() .expect("migration must fail") .to_string(); @@ -18014,7 +17740,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx fn runtime_binding_identity_and_trust_uniqueness_are_workspace_scoped() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18116,7 +17842,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); assert!( reopened .list_workspace_runtime_bindings("workspace-a", false) @@ -18166,7 +17892,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx fn workspace_runtime_connection_persists_without_authentication_evidence() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18211,7 +17937,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); assert!(store.workspace_runtime_binding_matches(&persisted).unwrap()); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); assert_eq!( reopened .get_workspace_runtime_binding("workspace-a", "runtime-a") @@ -18465,7 +18191,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn runtime_binding_key_mutations_are_identity_guarded_idempotent_and_audited() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18603,7 +18329,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn embedded_runtime_binding_can_explicitly_rotate_restart_identity() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -18703,7 +18429,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx conn.execute_batch("DROP TABLE typed_ticket_artifacts") .unwrap(); - let result = SqliteWorkspaceStore::from_connection_for_test(conn); + let result = SqliteWorkspaceStore::from_connection(conn); let error = match result { Ok(_) => panic!("schema drift unexpectedly passed startup verification"), Err(error) => error, @@ -18715,7 +18441,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn workspace_schema_rejects_cross_workspace_ticket_repository_reference_at_write_time() { let dir = tempfile::tempdir().unwrap(); let database_path = dir.path().join("workspace.sqlite"); - let store = SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); + let store = SqliteWorkspaceStore::open(&database_path).unwrap(); for workspace_id in ["workspace-a", "workspace-b"] { store .upsert_workspace(&WorkspaceRecord { @@ -18769,14 +18495,14 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(backend); - SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); + SqliteWorkspaceStore::open(&database_path).unwrap(); } #[tokio::test] async fn migrates_sqlite_and_preserves_workspace_record() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("control-plane.sqlite"); - let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); + let store = SqliteWorkspaceStore::open(&db).unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); assert!( @@ -18794,7 +18520,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx }; store.upsert_workspace(&record).await.unwrap(); - let reopened = SqliteWorkspaceStore::open_for_test(&db).unwrap(); + let reopened = SqliteWorkspaceStore::open(&db).unwrap(); assert_eq!( reopened.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION @@ -18809,7 +18535,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn workspace_display_name_update_is_timestamp_guarded_and_preserves_identity() { let dir = tempfile::tempdir().unwrap(); let database_path = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&database_path).unwrap(); + let store = SqliteWorkspaceStore::open(&database_path).unwrap(); let record = WorkspaceRecord { workspace_id: "workspace-a".to_string(), owner_account_id: "owner-account".to_string(), @@ -18848,7 +18574,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx updated ); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(database_path).unwrap(); + let reopened = SqliteWorkspaceStore::open(database_path).unwrap(); assert_eq!( reopened .get_workspace(&record.workspace_id) @@ -18862,7 +18588,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn objective_creation_allocates_and_resolves_workspace_resource_key() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".into(), @@ -18908,7 +18634,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn worker_create_reservation_allocates_uuid_before_runtime_and_replays_exact_input() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19212,7 +18938,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn worker_singleton_restart_retry_replays_exact_workdir_intent_and_rejects_changes() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); + let store = SqliteWorkspaceStore::open(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19272,7 +18998,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap(); + let reopened = SqliteWorkspaceStore::open(&database).unwrap(); let retried = reopened .reserve_worker_create( "workspace-a", @@ -19316,7 +19042,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn worker_singleton_ownership_is_workspace_scoped_durable_and_fences_old_generations() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); + let store = SqliteWorkspaceStore::open(&database).unwrap(); for workspace_id in ["workspace-a", "workspace-b"] { store .upsert_workspace(&WorkspaceRecord { @@ -19429,7 +19155,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap(); + let reopened = SqliteWorkspaceStore::open(database).unwrap(); assert_eq!( reopened .current_worker_singleton_owner("workspace-a", key) @@ -19442,7 +19168,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn concurrent_worker_singleton_reservations_have_one_winner() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); + let store = SqliteWorkspaceStore::open(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19462,7 +19188,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let database = database.clone(); let barrier = barrier.clone(); std::thread::spawn(move || { - let store = SqliteWorkspaceStore::open_for_test(database).unwrap(); + let store = SqliteWorkspaceStore::open(database).unwrap(); let memory = store.get_workspace_memory_settings("workspace-a").unwrap(); barrier.wait(); store.reserve_worker_create( @@ -19482,7 +19208,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .collect::>(); assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); assert_eq!(results.iter().filter(|result| result.is_err()).count(), 1); - let store = SqliteWorkspaceStore::open_for_test(database).unwrap(); + let store = SqliteWorkspaceStore::open(database).unwrap(); let owner = store .current_worker_singleton_owner("workspace-a", "shared-key") .unwrap() @@ -19498,7 +19224,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[tokio::test] async fn workspace_flow_sources_pin_content_and_builtins_stay_resources() { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); + let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -19591,7 +19317,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "returning to earlier content must reuse the immutable content" ); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(dir.path().join("server.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open(dir.path().join("server.db")).unwrap(); assert_eq!( reopened .get_flow_source_content( @@ -19847,7 +19573,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx } ); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "case-workspace").unwrap(); store .with_conn(|conn| { @@ -20014,7 +19740,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx ); drop(store); drop(backend); - let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); reopened .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); @@ -20081,7 +19807,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx before ); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); let backend = ticket::SqliteTicketBackend::open_verified(&path, "case-workspace").unwrap(); assert_eq!( backend @@ -20215,8 +19941,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("equivalence.db"); prepare_retained_schema(&path, 90); - let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); - let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory().unwrap(); for object in [ "workspace_tags", "typed_ticket_tags", @@ -20269,7 +19995,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx #[test] fn schema_v91_case_constraint_verification_rejects_legacy_view_predicates() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store.with_conn(|conn| { conn.execute_batch("DROP VIEW ticket_active_worker_assignments; CREATE VIEW ticket_active_worker_assignments AS @@ -20430,7 +20156,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx // boundary under test. Targets need not be materialized by a Runtime. async fn manual_worker_start_store() -> (tempfile::TempDir, SqliteWorkspaceStore) { let dir = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -20518,8 +20244,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx let threads = (0..2) .map(|index| { let store = - SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")) - .unwrap(); + SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); let barrier = barrier.clone(); std::thread::spawn(move || { barrier.wait(); @@ -20884,8 +20609,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .retain_ticket_claim_recovery("w", "claim", &fingerprint, &changed) .is_err() ); - let reopened = - SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); assert_eq!( reopened.get_ticket_claim_recovery("w", "claim").unwrap(), Some(recovery) @@ -20943,8 +20667,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx store .rollback_ticket_assignment_operation("w", "claim") .unwrap(); - let reopened = - SqliteWorkspaceStore::open_for_test(dir.path().join("workspace.db")).unwrap(); + let reopened = SqliteWorkspaceStore::open(dir.path().join("workspace.db")).unwrap(); let receipt = reopened .get_ticket_assignment_operation("w", "claim") .unwrap() @@ -21164,7 +20887,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx use ticket::TicketBackend; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "w".into(), @@ -21292,7 +21015,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx async fn ticket_worker_assignment_replaces_current_and_preserves_audit_history() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); + let store = SqliteWorkspaceStore::open(&db).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-a".to_string(), @@ -21481,7 +21204,7 @@ INSERT INTO worker_registry ( ) .unwrap(); drop(store); - let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); + let store = SqliteWorkspaceStore::open(&db).unwrap(); let pending = store .get_ticket_assignment_operation("workspace-a", "reserved-operation") .unwrap() @@ -21605,7 +21328,7 @@ INSERT INTO worker_registry ( async fn role_assignment_routes_and_manual_start_are_state_fenced_atomically() { let dir = tempfile::tempdir().unwrap(); let db_path = dir.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap(); + let store = SqliteWorkspaceStore::open(&db_path).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-role".to_string(), @@ -21894,7 +21617,7 @@ INSERT INTO worker_registry ( fn startup_rejects_missing_workspace_resource_constraint_trigger() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch("DROP TRIGGER ticket_worker_assignments_validate_insert")?; @@ -21903,7 +21626,7 @@ INSERT INTO worker_registry ( .unwrap(); drop(store); - let error = match SqliteWorkspaceStore::open_for_test(&path) { + let error = match SqliteWorkspaceStore::open(&path) { Ok(_) => panic!("missing assignment constraint trigger must fail closed"), Err(error) => error, }; @@ -21919,7 +21642,7 @@ INSERT INTO worker_registry ( fn startup_accepts_retained_assignment_history_after_parent_cleanup() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { conn.execute_batch( @@ -21983,7 +21706,7 @@ INSERT INTO worker_registry ( .unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); reopened .with_conn(|conn| { let retained: i64 = conn.query_row( @@ -22011,7 +21734,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn workspace_bootstrap_validates_key_and_rejects_duplicate_workspace() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store .upsert_account(&AccountRecord { account_id: "owner-account".to_string(), @@ -22143,7 +21866,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn repository_records_round_trip() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22224,7 +21947,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -22299,7 +22022,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn legacy_memory_reset_is_atomic_scoped_idempotent_and_exact() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); for (workspace_id, display_name) in [ ("local-dev", "Local Dev"), ("other-workspace", "Other Workspace"), @@ -22466,7 +22189,7 @@ INSERT INTO worker_registry ( async fn worker_workdir_registry_round_trips_and_preserves_pinned_retention() { let temp = tempfile::tempdir().unwrap(); let db = temp.path().join("workspace.db"); - let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); + let store = SqliteWorkspaceStore::open(&db).unwrap(); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: "owner-account".to_string(), @@ -22835,7 +22558,7 @@ INSERT INTO worker_registry ( ); } drop(store); - let store = SqliteWorkspaceStore::open_for_test(&db).unwrap(); + let store = SqliteWorkspaceStore::open(&db).unwrap(); assert_eq!( store .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 0, None) @@ -22865,7 +22588,7 @@ INSERT INTO worker_registry ( let workdir_conflict = store.attach_worker_workdir(&workdir_conflict).unwrap(); drop(store); - let reopened = SqliteWorkspaceStore::open_for_test(&db).unwrap(); + let reopened = SqliteWorkspaceStore::open(&db).unwrap(); assert_eq!( reopened .list_worker_workdir_links("local-dev", &worker.worker) @@ -22883,7 +22606,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn worker_projection_preserves_equal_local_ids_from_distinct_runtimes() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -23009,7 +22732,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn worker_projection_reconciles_snapshots_and_preserves_unavailable_lifecycle() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -23184,7 +22907,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn late_runtime_event_does_not_recreate_retention_deleted_catalog_worker() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")).unwrap(); + let store = SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -23279,7 +23002,7 @@ INSERT INTO worker_registry ( async fn worker_control_grants_are_idempotent_scoped_and_support_internal_invalidation() { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("control-grants.db"); - let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); + let store = SqliteWorkspaceStore::open(&database).unwrap(); store .upsert_workspace(&WorkspaceRecord { workspace_id: "workspace-control".to_string(), @@ -23349,7 +23072,7 @@ INSERT INTO worker_registry ( ); drop(store); - let store = SqliteWorkspaceStore::open_for_test(&database).unwrap(); + let store = SqliteWorkspaceStore::open(&database).unwrap(); assert_eq!( store .list_active_worker_control_grants( @@ -23407,7 +23130,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn account_and_login_records_round_trip() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { @@ -23766,9 +23489,9 @@ INSERT INTO worker_registry ( async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("resource-jobs.sqlite"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); seed_resource_job_workspace(&store).await; - let other = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let other = SqliteWorkspaceStore::open(&path).unwrap(); let requests = [ resource_job_request("first", "subject-a", "candidate-1"), resource_job_request("followup", "subject-a", "candidate-2"), @@ -23884,7 +23607,7 @@ INSERT INTO worker_registry ( async fn backend_job_resource_retry_and_unknown_are_fenced_by_durable_cleanup() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("cleanup-jobs.sqlite"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); seed_resource_job_workspace(&store).await; let request = resource_job_request("failed", "subject-a", "candidate-1"); let first = store @@ -23990,7 +23713,7 @@ INSERT INTO worker_registry ( .unwrap(); finish_resource_job_cleanup(&store, &second); drop(store); - let restarted = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let restarted = SqliteWorkspaceStore::open(&path).unwrap(); let active = restarted .find_active_backend_job_for_resource( "resource-workspace", @@ -24050,7 +23773,7 @@ INSERT INTO worker_registry ( async fn backend_job_resource_retry_races_new_intent_atomically_and_remains_bounded() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("retry-race.sqlite"); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); seed_resource_job_workspace(&store).await; // Generic resources use the same common reservation API without domain grants. let mut request = resource_job_request("generic", "subject-a", "candidate-1"); @@ -24073,7 +23796,7 @@ INSERT INTO worker_registry ( let mut followup = request.clone(); followup.job_id = "generic-followup".into(); followup.input = serde_json::json!({"candidate": "new"}); - let other = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let other = SqliteWorkspaceStore::open(&path).unwrap(); let barrier = std::sync::Barrier::new(2); let (retry, reserve) = std::thread::scope(|scope| { let a = scope.spawn(|| { @@ -24159,7 +23882,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_all_older_attempt_cleanups_fence_retry_and_resource_release() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); seed_resource_job_workspace(&store).await; let mut request = resource_job_request("legacy-overlap", "subject-a", "candidate-1"); request.limits.max_attempts = 3; @@ -24283,7 +24006,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_empty_batch_surface_refresh_locks_subject_and_defers_candidates() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); seed_resource_job_workspace(&store).await; for surface_state in ["ungenerated", "stale", "failed"] { let subject = format!("subject-{surface_state}"); @@ -24403,7 +24126,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_job_completed_resource_releases_only_after_worker_cleanup() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); seed_resource_job_workspace(&store).await; let request = resource_job_request("completed", "subject-a", "candidate-1"); let first = store @@ -24457,7 +24180,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn backend_jobs_fence_results_project_ownership_and_bound_retries() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); let workspace_id = "workspace-job"; store .upsert_account(&AccountRecord { diff --git a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs index 69190eb1f..0ea9c64e7 100644 --- a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs +++ b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs @@ -263,7 +263,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64 let tags_before = retained.as_ref().map(|_| classification_rows(&conn)); let auth_before = fixture_auth_rows(&conn); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store.with_conn(|conn| { assert_eq!(current_schema_version(conn)?,93); assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11); @@ -490,7 +490,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history let metadata_before = stored_fixture_rows(&conn, "typed_ticket_case_metadata"); let auth_before = fixture_auth_rows(&conn); drop(conn); - let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?,93); diff --git a/crates/workspace-server/src/workdir_create_operations.rs b/crates/workspace-server/src/workdir_create_operations.rs index a98284b1e..a41fc8ca7 100644 --- a/crates/workspace-server/src/workdir_create_operations.rs +++ b/crates/workspace-server/src/workdir_create_operations.rs @@ -750,7 +750,7 @@ mod tests { #[test] fn retry_keeps_resolved_config_and_credential_candidates_after_fallback_moves() { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord { workspace_id: "workspace".to_string(), owner_account_id: "owner-account".to_string(), diff --git a/crates/workspace-server/src/workdir_removal.rs b/crates/workspace-server/src/workdir_removal.rs index 13b481fb4..9c1f2a327 100644 --- a/crates/workspace-server/src/workdir_removal.rs +++ b/crates/workspace-server/src/workdir_removal.rs @@ -925,7 +925,7 @@ mod tests { } async fn seeded_store() -> (SqliteWorkspaceStore, WorkdirRegistryRecord) { - let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let store = SqliteWorkspaceStore::in_memory().unwrap(); store .upsert_account(&AccountRecord { account_id: "account-a".to_string(), diff --git a/crates/workspace-server/src/worker_projection.rs b/crates/workspace-server/src/worker_projection.rs index fc8140132..ed910de04 100644 --- a/crates/workspace-server/src/worker_projection.rs +++ b/crates/workspace-server/src/worker_projection.rs @@ -5,7 +5,7 @@ use chrono::Utc; use protocol::subscription::{ EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot, }; -use tokio::sync::{Semaphore, broadcast}; +use tokio::sync::broadcast; use tokio::task::JoinHandle; use crate::runtime_subscription::{ @@ -59,14 +59,7 @@ pub struct WorkerProjectionService { events: broadcast::Sender, ordering: Mutex<()>, attach_gate: tokio::sync::Mutex<()>, - tasks: Mutex>, -} - -struct RuntimeProjectionLifetime { - consumer: Option>, - // The permit belongs to the actual DB closure, not its cancellable async waiter. - // Keep this owner in the map even if attachment initialization is cancelled. - operations: Arc, + tasks: Mutex>>, } impl WorkerProjectionService { @@ -82,89 +75,17 @@ impl WorkerProjectionService { }) } - /// Stops consumers but does not claim that accepted database work has finished. - /// Async shutdown callers must use `shutdown_async` to drain accepted operations. pub fn shutdown(&self) { - for lifetime in self + for (_, task) in self .tasks .lock() .expect("Worker projection task lock poisoned") - .values() + .drain() { - if let Some(task) = &lifetime.consumer { - task.abort(); - } + task.abort(); } } - pub async fn shutdown_async(&self) { - let _attach = self.attach_gate.lock().await; - self.shutdown(); - let runtime_ids = self - .tasks - .lock() - .expect("Worker projection task lock poisoned") - .keys() - .cloned() - .collect::>(); - for runtime_id in runtime_ids { - self.drain_runtime(&runtime_id).await; - } - self.tasks - .lock() - .expect("Worker projection task lock poisoned") - .clear(); - } - - async fn drain_runtime(&self, runtime_id: &str) { - let previous = self - .tasks - .lock() - .expect("Worker projection task lock poisoned") - .get_mut(runtime_id) - .map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone())); - if let Some((consumer, operations)) = previous { - if let Some(consumer) = consumer { - consumer.abort(); - let _ = consumer.await; - } - // Aborting the waiter cannot abort a queued/running synchronous job. - // Cancellation here leaves the gate in the map for the next drain. - let _drained = operations - .acquire_owned() - .await - .expect("operation gate closed"); - } - } - - // Caller holds attach_gate. Install ownership before any initialization await. - async fn replace_runtime(&self, runtime_id: &str) -> Arc { - self.drain_runtime(runtime_id).await; - let operations = Arc::new(Semaphore::new(1)); - self.tasks - .lock() - .expect("Worker projection task lock poisoned") - .insert( - runtime_id.to_string(), - RuntimeProjectionLifetime { - consumer: None, - operations: operations.clone(), - }, - ); - operations - } - - pub async fn subscribe_ordered_async( - self: &Arc, - limit: usize, - ) -> crate::Result { - let service = self.clone(); - self.store - .db_executor() - .run(move || service.subscribe_ordered(limit)) - .await - } - pub fn subscribe_ordered(&self, limit: usize) -> crate::Result { let _ordering = self .ordering @@ -204,10 +125,19 @@ impl WorkerProjectionService { broker: &RuntimeSubscriptionBroker, runtime_id: &str, ) -> Result<(), RuntimeSubscriptionBrokerError> { - // Serialize replacement, stop admission by aborting the consumer, and then - // drain ownership held by the actual SQL jobs before the new snapshot. + // Serialize replacement and wait for the old consumer to stop before the new + // lifetime can apply its snapshot. Late notifications can therefore never + // cross a Runtime ownership boundary. let _attach = self.attach_gate.lock().await; - let operations = self.replace_runtime(runtime_id).await; + let previous = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .remove(runtime_id); + if let Some(previous) = previous { + previous.abort(); + let _ = previous.await; + } let mut subscription = broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?; @@ -218,25 +148,12 @@ impl WorkerProjectionService { break; }; let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. }); - if let Err(error) = self - .apply_broker_event_owned_async(runtime_id, event, operations.clone()) - .await - { + if let Err(error) = self.apply_broker_event(runtime_id, event) { tracing::warn!( runtime_id, error = %error, "failed to initialize durable Worker projection; restarting from a fresh snapshot" ); - if matches!( - error, - crate::Error::DbOperationOutcomeUnknown - | crate::Error::DbExecutorUnavailable - | crate::Error::DbCapacityExceeded - ) { - // Do not automatically replay a possibly committed mutation or - // spin on executor overload; attachment must be retried explicitly. - return Err(RuntimeSubscriptionBrokerError::Closed); - } subscription.restart()?; continue; } @@ -252,27 +169,12 @@ impl WorkerProjectionService { let Some(service) = service.upgrade() else { return; }; - if let Err(error) = service - .apply_broker_event_owned_async( - task_runtime_id.as_str(), - event, - operations.clone(), - ) - .await - { + if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) { tracing::warn!( runtime_id = %task_runtime_id, error = %error, "failed to update durable Worker projection; restarting from a fresh snapshot" ); - if matches!( - error, - crate::Error::DbOperationOutcomeUnknown - | crate::Error::DbExecutorUnavailable - | crate::Error::DbCapacityExceeded - ) { - return; - } if subscription.restart().is_err() { return; } @@ -282,61 +184,10 @@ impl WorkerProjectionService { self.tasks .lock() .expect("Worker projection task lock poisoned") - .get_mut(&runtime_id) - .expect("Runtime lifetime installed before initialization") - .consumer = Some(task); + .insert(runtime_id, task); Ok(()) } - #[cfg(test)] - async fn seed_observation_async( - self: &Arc, - runtime_id: &str, - worker: &protocol::subscription::SubscriptionWorker, - observed_at: &str, - ) -> crate::Result<()> { - let service = self.clone(); - let runtime_id = runtime_id.to_string(); - let worker = worker.clone(); - let observed_at = observed_at.to_string(); - self.store - .db_executor() - .run(move || service.seed_observation(&runtime_id, &worker, &observed_at)) - .await - } - - pub async fn refresh_async(self: &Arc, worker: &RuntimeWorkerRef) -> crate::Result<()> { - let service = self.clone(); - let worker = worker.clone(); - self.store - .db_executor() - .run(move || service.refresh(&worker)) - .await - } - - async fn apply_broker_event_owned_async( - self: &Arc, - runtime_id: &str, - event: BrokerSubscriptionEvent, - operations: Arc, - ) -> crate::Result<()> { - let operation = operations - .acquire_owned() - .await - .expect("operation gate closed"); - let service = self.clone(); - let runtime_id = runtime_id.to_string(); - self.store - .db_executor() - .run(move || { - // The permit must outlive SQL commit AND ordered publication, even - // when the executor result receiver has been dropped by cancellation. - let _operation = operation; - service.apply_broker_event(&runtime_id, event) - }) - .await - } - pub fn seed_observation( &self, runtime_id: &str, @@ -462,11 +313,7 @@ mod tests { tempfile::TempDir, ) { let temp = tempfile::tempdir().unwrap(); - let store = Arc::new( - SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db")) - .unwrap() - .with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()), - ); + let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap()); store .upsert_account(&AccountRecord { account_id: "owner".to_string(), @@ -508,266 +355,6 @@ mod tests { } } - fn observation( - state: protocol::subscription::SubscriptionWorkerState, - ) -> protocol::subscription::SubscriptionWorker { - protocol::subscription::SubscriptionWorker { - restore_observation_token: None, - worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(), - runtime_id: Some("runtime-a".to_string()), - resource_key: None, - availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, - worker_state: None, - state, - has_running_internal_workers: false, - workspace_id: Some(WORKSPACE_ID.to_string()), - display_name: None, - profile: None, - job: None, - workdir_attachments: Vec::new(), - } - } - - // Real second-connection contention; the controller releases automatically on - // failure, so a regression that blocks the Tokio thread cannot hang the suite. - struct WriteLock { - release: Option>, - controller: Option>, - } - - impl WriteLock { - fn acquire(path: std::path::PathBuf) -> Self { - let (release, released) = std::sync::mpsc::channel(); - let (locked, ready) = std::sync::mpsc::channel(); - let controller = std::thread::spawn(move || { - let connection = rusqlite::Connection::open(path).unwrap(); - connection.execute_batch("BEGIN IMMEDIATE").unwrap(); - locked.send(()).unwrap(); - let _ = released.recv_timeout(std::time::Duration::from_secs(3)); - connection.execute_batch("ROLLBACK").unwrap(); - }); - ready.recv().unwrap(); - Self { - release: Some(release), - controller: Some(controller), - } - } - - fn release(&mut self) { - if let Some(release) = self.release.take() { - let _ = release.send(()); - } - } - } - - impl Drop for WriteLock { - fn drop(&mut self) { - self.release(); - if let Some(controller) = self.controller.take() { - controller.join().unwrap(); - } - } - } - - async fn wait_for_sql(service: &WorkerProjectionService) { - tokio::time::timeout(std::time::Duration::from_secs(1), async { - // With the second connection holding the write lock, a held ordering - // gate identifies the accepted synchronous mutation, not a fake future. - while service.ordering.try_lock().is_ok() { - tokio::task::yield_now().await; - } - }) - .await - .expect("projection job did not enter synchronous SQL"); - } - - #[tokio::test(flavor = "current_thread")] - async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload() - { - use protocol::subscription::SubscriptionWorkerState; - let (store, service, temp) = fixture().await; - store - .upsert_worker_registry(&worker("runtime-a", "worker-a")) - .unwrap(); - let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); - assert_eq!(subscription.take_snapshot().len(), 1); - let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); - let publishing_service = service.clone(); - let publication = tokio::spawn(async move { - publishing_service - .seed_observation_async( - "runtime-a", - &observation(SubscriptionWorkerState::Idle), - "2", - ) - .await - }); - wait_for_sql(&service).await; - let started = std::time::Instant::now(); - let (control, controlled) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - control.send("control progressed").unwrap(); - }); - let (_, control_result) = tokio::join!( - tokio::time::sleep(std::time::Duration::from_millis(20)), - controlled, - ); - assert_eq!(control_result.unwrap(), "control progressed"); - assert!( - started.elapsed() < std::time::Duration::from_secs(1), - "SQLite blocked the current-thread executor" - ); - assert!(matches!( - subscription.try_recv(), - Err(tokio::sync::broadcast::error::TryRecvError::Empty) - )); - lock.release(); - publication.await.unwrap().unwrap(); - let committed = subscription.recv().await.unwrap(); - let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); - let snapshot = snapshot.take_snapshot(); - assert_eq!( - committed.changes, - vec![WorkerCatalogChange::Upsert(snapshot[0].clone())] - ); - assert_eq!( - snapshot[0].observation.as_ref().unwrap().worker.state, - SubscriptionWorkerState::Idle - ); - service - .seed_observation_async( - "runtime-a", - &observation(SubscriptionWorkerState::Running), - "3", - ) - .await - .unwrap(); - // Delayed readers receive the first transaction's payload, not a re-query - // of mutable authority after a later commit. - let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else { - panic!("expected upsert"); - }; - assert_eq!( - first.observation.as_ref().unwrap().worker.state, - SubscriptionWorkerState::Idle - ); - service - .refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a")) - .await - .unwrap(); - } - - #[tokio::test(flavor = "current_thread")] - async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() { - use crate::runtime_subscription::BrokerSubscriptionEvent; - use protocol::subscription::SubscriptionWorkerState; - let (store, service, temp) = fixture().await; - store - .upsert_worker_registry(&worker("runtime-a", "worker-a")) - .unwrap(); - let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); - subscription.take_snapshot(); - let operations = service.replace_runtime("runtime-a").await; - let old_operations = operations.clone(); - let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); - let old_service = service.clone(); - let consumer = tokio::spawn(async move { - old_service - .apply_broker_event_owned_async( - "runtime-a", - BrokerSubscriptionEvent::Event { - payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted { - worker: observation(SubscriptionWorkerState::Running), - }, - }, - operations, - ) - .await - .unwrap(); - }); - service - .tasks - .lock() - .unwrap() - .get_mut("runtime-a") - .unwrap() - .consumer = Some(consumer); - wait_for_sql(&service).await; - let replacing_service = service.clone(); - let mut replacement = tokio::spawn(async move { - let _attach = replacing_service.attach_gate.lock().await; - replacing_service.replace_runtime("runtime-a").await - }); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) - .await - .is_err() - ); - replacement.abort(); - assert!(replacement.await.unwrap_err().is_cancelled()); - assert_eq!( - old_operations.available_permits(), - 0, - "cancelled drain lost accepted SQL ownership" - ); - // A second replacement must still observe the old gate after cancellation. - let replacing_service = service.clone(); - let mut replacement = tokio::spawn(async move { - let _attach = replacing_service.attach_gate.lock().await; - let operations = replacing_service.replace_runtime("runtime-a").await; - replacing_service - .apply_broker_event_owned_async( - "runtime-a", - BrokerSubscriptionEvent::Snapshot { - snapshot: protocol::subscription::SubscriptionSnapshot::Workers { - workers: vec![observation(SubscriptionWorkerState::Idle)], - }, - }, - operations, - ) - .await - }); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) - .await - .is_err() - ); - assert!(matches!( - subscription.try_recv(), - Err(tokio::sync::broadcast::error::TryRecvError::Empty) - )); - lock.release(); - replacement.await.unwrap().unwrap(); - assert_eq!(old_operations.available_permits(), 1); - let old_commit = subscription.recv().await.unwrap(); - let new_commit = subscription.recv().await.unwrap(); - for (commit, expected) in [ - (old_commit, SubscriptionWorkerState::Running), - (new_commit, SubscriptionWorkerState::Idle), - ] { - let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else { - panic!("expected one committed upsert"); - }; - assert_eq!(record.observation.as_ref().unwrap().worker.state, expected); - } - let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); - assert_eq!( - snapshot.take_snapshot()[0] - .observation - .as_ref() - .unwrap() - .worker - .state, - SubscriptionWorkerState::Idle - ); - assert!(matches!( - subscription.try_recv(), - Err(tokio::sync::broadcast::error::TryRecvError::Empty) - )); - service.shutdown_async().await; - assert!(service.tasks.lock().unwrap().is_empty()); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn snapshot_registration_is_atomic_with_concurrent_publication() { let (store, service, _temp) = fixture().await; diff --git a/crates/workspace-server/src/worker_source.rs b/crates/workspace-server/src/worker_source.rs index 3edbb5d3e..3d51476d1 100644 --- a/crates/workspace-server/src/worker_source.rs +++ b/crates/workspace-server/src/worker_source.rs @@ -70,24 +70,10 @@ pub enum WorkerMutationSourceProofError { Replay, #[error("source Worker is not a current member of this Workspace Runtime catalog")] WorkerCatalogMembership, - #[error("source proof authority is unavailable")] - Unavailable, #[error("source proof authority failed: {0}")] Authority(String), } -impl From for WorkerMutationSourceProofError { - fn from(error: crate::Error) -> Self { - match error { - crate::Error::DbCapacityExceeded - | crate::Error::DbExecutorUnavailable - | crate::Error::DbOperationOutcomeUnknown - | crate::Error::Sqlite(_) => Self::Unavailable, - other => Self::Authority(other.to_string()), - } - } -} - pub async fn verify_worker_remove_source( api: &WorkspaceApi, proof: PresentedWorkerMutationSourceProof<'_>, @@ -121,7 +107,7 @@ async fn verify_worker_remove_source_with( let trusted = store .get_workspace_runtime_binding(&config.workspace_id, &unverified.iss) .await - .map_err(WorkerMutationSourceProofError::from)? + .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))? .filter(|record| record.revoked_at.is_none()) .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?; let public_key = trusted.public_key.as_str(); @@ -165,23 +151,15 @@ async fn verify_worker_remove_source_with( runtime_id: claims.iss.clone(), worker_id: claims.worker_id.clone(), }; - let workspace_id = config.workspace_id.clone(); - let catalog_store = Arc::clone(store); + let member = store + .get_worker_registry(&config.workspace_id, &worker) + .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + if member.is_none() { + return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); + } store - .db_executor() - .run(move || { - let member = catalog_store - .get_worker_registry(&workspace_id, &worker) - .map_err(WorkerMutationSourceProofError::from)?; - if member.is_none() { - return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); - } - catalog_store - .require_current_worker_singleton_owner(&workspace_id, &worker) - .map_err(WorkerMutationSourceProofError::from)?; - Ok(()) - }) - .await?; + .require_current_worker_singleton_owner(&config.workspace_id, &worker) + .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; let consumed_at = chrono::Utc::now().to_rfc3339(); let consumed = store @@ -194,7 +172,7 @@ async fn verify_worker_remove_source_with( &consumed_at, ) .await - .map_err(WorkerMutationSourceProofError::from)?; + .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; if !consumed { return Err(WorkerMutationSourceProofError::Replay); } diff --git a/crates/workspace-server/src/worker_source/runtime_request.rs b/crates/workspace-server/src/worker_source/runtime_request.rs index b8c2e7226..9e79fd3b1 100644 --- a/crates/workspace-server/src/worker_source/runtime_request.rs +++ b/crates/workspace-server/src/worker_source/runtime_request.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use worker_runtime::auth::{ RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims, RuntimeRequestSourceExpectation, decode_runtime_request_source_claims, @@ -35,21 +33,9 @@ pub struct RuntimeRequestProofError { pub(crate) exp: Option, pub(crate) now: Option, pub(crate) time_delta_seconds: Option, - /// Operational unavailability is not a proof or principal rejection. - pub(crate) availability: bool, } impl RuntimeRequestProofError { - pub fn is_availability_error(&self) -> bool { - self.availability - } - - fn unavailable(mut self, reason: &'static str) -> Self { - self.reason = reason; - self.availability = true; - self - } - fn undecoded() -> Self { Self { reason: "invalid", @@ -61,7 +47,6 @@ impl RuntimeRequestProofError { exp: None, now: None, time_delta_seconds: None, - availability: false, } } @@ -121,18 +106,6 @@ impl RuntimeRequestProofError { } } -impl From for RuntimeRequestProofError { - fn from(error: crate::Error) -> Self { - let reason = match error { - crate::Error::DbCapacityExceeded => "db_capacity_exceeded", - crate::Error::DbExecutorUnavailable => "db_executor_unavailable", - crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown", - _ => "database_authority_unavailable", - }; - Self::undecoded().unavailable(reason) - } -} - /// Domain-separated fixed-length fingerprints, not raw request-derived IDs. /// Operators can correlate a known ID by applying this same function. pub fn diagnostic_id_hash(id: &str) -> String { @@ -157,7 +130,7 @@ pub async fn verify_runtime_request_source_proof( body_digest: &str, ) -> Result { verify_runtime_request_source_proof_with_store( - Arc::clone(&api.store), + api.store.as_ref(), &api.config, proof, workspace_id, @@ -171,7 +144,7 @@ pub async fn verify_runtime_request_source_proof( #[allow(clippy::too_many_arguments)] pub async fn verify_runtime_request_source_proof_with_store( - store: Arc, + store: &dyn ControlPlaneStore, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -196,7 +169,7 @@ pub async fn verify_runtime_request_source_proof_with_store( #[allow(clippy::too_many_arguments)] pub(crate) async fn verify_runtime_request_source_proof_with_clock( - store: Arc, + store: &dyn ControlPlaneStore, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -209,18 +182,15 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( let unverified = decode_runtime_request_source_claims(proof) .map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?; let diagnostic = RuntimeRequestProofError::decoded(&unverified); - let audience = remote_audience(config, workspace_id).map_err(|_| { - diagnostic - .clone() - .unavailable("backend_audience_unavailable") - })?; + let audience = remote_audience(config, workspace_id) + .map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?; let trusted = store .get_workspace_runtime_binding(workspace_id, &unverified.iss) .await .map_err(|_| { diagnostic .clone() - .unavailable("runtime_trust_authority_unavailable") + .reason("runtime_trust_authority_unavailable") })? .filter(|record| record.revoked_at.is_none()) .ok_or_else(|| { @@ -268,11 +238,7 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( &consumed_at, ) .await - .map_err(|_| { - diagnostic - .clone() - .unavailable("replay_authority_unavailable") - })? + .map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))? { return Err(diagnostic.reason("replay")); } @@ -281,58 +247,29 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( runtime_id: claims.iss.clone(), worker_id: worker_id.to_owned(), }; - let workspace_id = workspace_id.to_owned(); - let executor = store.db_executor(); - let stage_diagnostic = diagnostic.clone(); - // Only synchronous catalog authority lives in this closure. The trust - // and JTI async wrappers above already offload through the executor. - executor - .run(move || { - let member = store - .get_worker_registry(&workspace_id, &worker) - .map_err(|_| { - stage_diagnostic - .clone() - .unavailable("worker_catalog_authority_unavailable") - })?; - let reserved = store - .has_active_worker_create_reservation(&workspace_id, &worker) - .map_err(|_| { - stage_diagnostic - .clone() - .unavailable("worker_reservation_authority_unavailable") - })?; - if member.is_none() && !reserved { - return Err(stage_diagnostic.reason("worker_catalog_membership")); - } - store - .require_current_worker_singleton_owner(&workspace_id, &worker) - .map_err(|error| { - if matches!( - error, - crate::Error::DbCapacityExceeded - | crate::Error::DbExecutorUnavailable - | crate::Error::DbOperationOutcomeUnknown - | crate::Error::Sqlite(_) - ) { - stage_diagnostic - .clone() - .unavailable("worker_singleton_authority_unavailable") - } else { - stage_diagnostic - .clone() - .reason("worker_singleton_authority_denied") - } - })?; - Ok(()) - }) - .await - .map_err(|error: RuntimeRequestProofError| { - if error.verification == "unverified" && error.is_availability_error() { - diagnostic.clone().unavailable(error.reason) - } else { - error - } + let member = store + .get_worker_registry(workspace_id, &worker) + .map_err(|_| { + diagnostic + .clone() + .reason("worker_catalog_authority_unavailable") + })?; + let reserved = store + .has_active_worker_create_reservation(workspace_id, &worker) + .map_err(|_| { + diagnostic + .clone() + .reason("worker_reservation_authority_unavailable") + })?; + if member.is_none() && !reserved { + return Err(diagnostic.reason("worker_catalog_membership")); + } + store + .require_current_worker_singleton_owner(workspace_id, &worker) + .map_err(|_| { + diagnostic + .clone() + .reason("worker_singleton_authority_denied") })?; } Ok(VerifiedRuntimeRequestSource { @@ -342,290 +279,3 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( authentication_binding_id: trusted.binding_id, }) } - -#[cfg(test)] -mod tests { - use super::*; - use crate::store::{ - AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode, - WorkspaceRuntimeBinding, - }; - use std::time::Duration; - use worker_runtime::auth::{ - BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner, - request_body_digest, - }; - - const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa"; - const NOW: i64 = 1_700_000_000; - const PATH: &str = "/api/runtime/v1/resources/fetch"; - - // rusqlite's busy handler accepts a function pointer, not a closure. Each - // fixture owns a dedicated single-worker executor, so this thread-local - // handshake is isolated from all parallel tests and product connections. - thread_local! { - static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell, - std::sync::mpsc::Receiver<()>, - )>> = const { std::cell::RefCell::new(None) }; - } - - fn acknowledge_sqlite_contention(_attempt: i32) -> bool { - SQLITE_BUSY_HANDSHAKE.with(|slot| { - let Some((blocked, retry)) = slot.borrow_mut().take() else { - return false; - }; - let _ = blocked.send(()); - // The test releases SQLite's writer lock before allowing retry. - // Dropping the sender on panic also releases this worker promptly. - retry.recv_timeout(Duration::from_secs(3)).is_ok() - }) - } - - async fn fixture( - root: &std::path::Path, - ) -> (Arc, ServerConfig, String, String) { - let db_path = root.join("auth.db"); - let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { - workers: std::num::NonZeroUsize::new(1).unwrap(), - queued: std::num::NonZeroUsize::new(1).unwrap(), - }) - .unwrap(); - let store = Arc::new( - SqliteWorkspaceStore::open_for_test(&db_path) - .unwrap() - .with_db_executor(executor), - ); - store - .upsert_account(&AccountRecord { - account_id: "account-auth".into(), - kind: "user".into(), - handle: "auth".into(), - display_name: "Auth".into(), - created_at: "1".into(), - updated_at: "1".into(), - }) - .unwrap(); - let workspace = WorkspaceRecord { - workspace_id: WORKSPACE.into(), - owner_account_id: "account-auth".into(), - display_name: "Auth".into(), - state: "active".into(), - created_at: "1".into(), - updated_at: "1".into(), - }; - store.upsert_workspace(&workspace).await.unwrap(); - let config = ServerConfig { - workspace_id: WORKSPACE.into(), - workspace_display_name: "Auth".into(), - workspace_created_at: "1".into(), - workspace_execution_root: root.to_owned(), - database_path: db_path, - data_root: root.to_owned(), - embedded_runtime_store_root: root.join("runtime"), - static_assets_dir: None, - auth: crate::server::AuthConfig::Passkey { - rp_id: "localhost".into(), - origin: "http://localhost".into(), - public_base_url: "http://localhost".into(), - cookie_name: "session".into(), - }, - max_records: 10, - repositories: vec![], - runtime_event_sources: vec![], - remote_runtime_sources: vec![], - backend_base_url: Some("server-auth".into()), - }; - let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap(); - store - .upsert_workspace_runtime_binding_record( - WorkspaceRuntimeBinding { - workspace_id: WORKSPACE.into(), - runtime_id: "runtime-auth".into(), - display_name: "Auth".into(), - base_url: "http://127.0.0.1:1".into(), - public_key: identity.public_key.clone(), - public_key_fingerprint: String::new(), - binding_id: String::new(), - authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer, - created_at: "1".into(), - updated_at: "1".into(), - revoked_at: None, - }, - false, - ) - .await - .unwrap(); - let binding = ControlPlaneStore::get_workspace_runtime_binding( - store.as_ref(), - WORKSPACE, - "runtime-auth", - ) - .await - .unwrap() - .unwrap(); - let proof = RuntimeRequestSourceSigner::from_identity(&identity) - .issue( - "server-auth", - WORKSPACE, - None, - BACKEND_RESOURCE_FETCH_PERMISSION, - "POST", - PATH, - b"{}", - NOW, - 30, - ) - .unwrap(); - (store, config, proof, binding.binding_id) - } - - #[tokio::test(flavor = "current_thread")] - async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() { - let root = tempfile::tempdir().unwrap(); - let (store, config, proof, binding_id) = fixture(root.path()).await; - // A real independent SQLite writer prevents JTI insertion while trust - // reads remain possible. Rollback on Drop also releases this on panic. - let mut writer = rusqlite::Connection::open(&config.database_path).unwrap(); - let lock = writer - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .unwrap(); - let (blocked, contention_reached) = tokio::sync::oneshot::channel(); - let (retry, retry_rx) = std::sync::mpsc::channel(); - let setup_store = store.clone(); - store - .db_executor() - .run(move || -> crate::Result<()> { - SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx))); - setup_store.with_conn(|conn| { - conn.busy_handler(Some(acknowledge_sqlite_contention))?; - Ok(()) - }) - }) - .await - .unwrap(); - let request_store = store.clone(); - let request_config = config.clone(); - let request_proof = proof.clone(); - let task = tokio::spawn(async move { - verify_runtime_request_source_proof_with_clock( - request_store, - &request_config, - &request_proof, - WORKSPACE, - BACKEND_RESOURCE_FETCH_PERMISSION, - "POST", - PATH, - &request_body_digest(b"{}"), - &|| NOW, - ) - .await - }); - tokio::time::timeout(Duration::from_secs(1), contention_reached) - .await - .expect( - "SQLite JTI persistence must report real writer contention without blocking Tokio", - ) - .unwrap(); - // The busy callback acknowledges an actual SQLite write blocked by the - // independent writer, then holds retry until explicitly released. - assert!( - !task.is_finished(), - "JTI must remain pending while SQLite writer is locked" - ); - let control = tokio::spawn(async { "control-responsive" }); - assert_eq!( - tokio::time::timeout(Duration::from_secs(1), control) - .await - .unwrap() - .unwrap(), - "control-responsive" - ); - // This timer is the behavior being tested, not a guessed wait for SQL. - tokio::time::timeout( - Duration::from_secs(1), - tokio::time::sleep(Duration::from_millis(1)), - ) - .await - .unwrap(); - assert!( - !task.is_finished(), - "timer must advance while JTI remains blocked" - ); - lock.rollback().unwrap(); - retry.send(()).unwrap(); - let source = tokio::time::timeout(Duration::from_secs(3), task) - .await - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(source.runtime_id, "runtime-auth"); - assert_eq!(source.authentication_binding_id, binding_id); - let replay = verify_runtime_request_source_proof_with_clock( - store, - &config, - &proof, - WORKSPACE, - BACKEND_RESOURCE_FETCH_PERMISSION, - "POST", - PATH, - &request_body_digest(b"{}"), - &|| NOW, - ) - .await - .unwrap_err(); - assert_eq!(replay.reason, "replay"); - assert!(!replay.is_availability_error()); - } - - #[tokio::test] - async fn saturated_database_is_availability_failure_not_invalid_proof() { - let root = tempfile::tempdir().unwrap(); - let (store, config, proof, _) = fixture(root.path()).await; - // Fill the isolated executor with a blocked job and queued jobs. No - // scheduling sleeps: the worker explicitly acknowledges its start. - let executor = store.db_executor(); - let (started, started_rx) = tokio::sync::oneshot::channel(); - let (release, release_rx) = std::sync::mpsc::channel(); - let blocker_executor = executor.clone(); - let blocker = tokio::spawn(async move { - blocker_executor - .run(move || -> crate::Result<()> { - let _ = started.send(()); - release_rx.recv().unwrap(); - Ok(()) - }) - .await - }); - tokio::time::timeout(Duration::from_secs(1), started_rx) - .await - .unwrap() - .unwrap(); - let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) })); - assert!(futures::poll!(queued.as_mut()).is_pending()); - let outcome = tokio::time::timeout( - Duration::from_secs(1), - verify_runtime_request_source_proof_with_clock( - store, - &config, - &proof, - WORKSPACE, - BACKEND_RESOURCE_FETCH_PERMISSION, - "POST", - PATH, - &request_body_digest(b"{}"), - &|| NOW, - ), - ) - .await; - // Release before assertions so a failed contract never strands a worker. - release.send(()).unwrap(); - blocker.await.unwrap().unwrap(); - queued.await.unwrap(); - let error = outcome - .expect("capacity rejection must not wait for queued SQL") - .unwrap_err(); - assert_eq!(error.reason, "runtime_trust_authority_unavailable"); - assert!(error.is_availability_error()); - } -} diff --git a/crates/workspace-server/src/workspace_catalog.rs b/crates/workspace-server/src/workspace_catalog.rs index 78ce46f45..5d9679d9f 100644 --- a/crates/workspace-server/src/workspace_catalog.rs +++ b/crates/workspace-server/src/workspace_catalog.rs @@ -285,7 +285,7 @@ mod tests { #[tokio::test] async fn create_is_atomic_and_exact_retries_converge() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); let service = WorkspaceCatalogService::new( store.clone(), Arc::new( @@ -339,7 +339,7 @@ mod tests { #[test] fn create_rejects_noncanonical_repository_keys_without_persisting_a_workspace() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); let service = WorkspaceCatalogService::new( store.clone(), Arc::new( @@ -378,7 +378,7 @@ mod tests { for with_repository in [false, true] { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); let owner_account_id = owner_account(store.as_ref()); let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default()); let service = WorkspaceCatalogService::new( @@ -418,7 +418,7 @@ mod tests { drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); let restarted = WorkspaceCatalogService::new(store.clone(), materials); let created = restarted.create(request, owner_account_id).unwrap(); let identity = store @@ -435,7 +435,7 @@ mod tests { for with_repository in [false, true] { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); let owner_account_id = owner_account(store.as_ref()); let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default()); let service = WorkspaceCatalogService::new(store.clone(), materials.clone()); @@ -486,7 +486,7 @@ mod tests { drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); let restarted = WorkspaceCatalogService::new(store.clone(), materials); let created = restarted.create(request, owner_account_id).unwrap(); let identity = store @@ -500,7 +500,7 @@ mod tests { #[tokio::test] async fn idempotency_key_reuse_with_different_payload_is_rejected() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); let owner_account_id = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store, @@ -532,7 +532,7 @@ mod tests { #[test] fn repository_free_create_initializes_authority_and_concurrent_retries_converge() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), @@ -611,7 +611,7 @@ mod tests { #[test] fn replay_reads_current_repository_at_initial_key_and_never_recreates_it() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), @@ -673,13 +673,13 @@ mod tests { repository: None, }; let (owner, created) = { - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); let owner = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new(store, materials.clone()); let created = service.create(request.clone(), owner.clone()).unwrap(); (owner, created) }; - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap()); let reopened = WorkspaceCatalogService::new(store.clone(), materials); let replay = reopened.create(request, owner).unwrap(); assert_eq!(replay.workspace, created.workspace); @@ -739,7 +739,7 @@ mod tests { #[test] fn create_rejects_non_user_account_owners() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); store .upsert_account(&AccountRecord { account_id: "organization-owner".to_string(), @@ -777,7 +777,7 @@ mod tests { #[test] fn catalog_list_is_scoped_to_the_required_owner_account() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); let owner_a = owner_account(store.as_ref()); let owner_b = "account-owner-b".to_string(); store @@ -845,7 +845,7 @@ mod tests { #[test] fn remote_repository_creation_persists_typed_source_without_auth_metadata() { - let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap()); + let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap()); let owner_account_id = owner_account(store.as_ref()); let service = WorkspaceCatalogService::new( store.clone(), diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index 5003d4153..e241f8aee 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -236,7 +236,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { "Workspace deletion request exceeds bounded field limits".to_string(), )); } - self.with_immediate_transaction(|tx| { + self.with_transaction(|tx| { if let Some(existing) = read_operation(tx, &request.operation_id)? { if existing.actor_account_id != actor_account_id { return Err(Error::WorkspacePermissionDenied( @@ -369,7 +369,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { child_operation_id: &str, ) -> Result { validate_operation_id(child_operation_id)?; - self.with_immediate_transaction(|tx| { + self.with_transaction(|tx| { let operation = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? .response; @@ -411,7 +411,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { failure_category: Option<&str>, ) -> Result { validate_operation_projection(child_operation_ids, blockers)?; - self.with_immediate_transaction(|tx| { + self.with_transaction(|tx| { let current = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? .response; @@ -462,7 +462,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { &self, operation_id: &str, ) -> Result { - self.with_immediate_transaction(|tx| { + self.with_transaction(|tx| { tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?; let operation = read_operation(tx, operation_id)? .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; @@ -922,7 +922,7 @@ mod tests { use tempfile::tempdir; fn setup() -> (SqliteWorkspaceStore, String, String) { - seed_workspaces(SqliteWorkspaceStore::in_memory_for_test().expect("store")) + seed_workspaces(SqliteWorkspaceStore::in_memory().expect("store")) } fn seed_workspaces(store: SqliteWorkspaceStore) -> (SqliteWorkspaceStore, String, String) { @@ -989,7 +989,7 @@ mod tests { let dir = tempdir().expect("tempdir"); let path = dir.path().join("server.db"); let (store, owner, workspace_id) = - seed_workspaces(SqliteWorkspaceStore::open_for_test(&path).expect("store")); + seed_workspaces(SqliteWorkspaceStore::open(&path).expect("store")); for workspace in ["workspace-a", "workspace-b"] { let backend = ticket::SqliteTicketBackend::open_verified(&path, workspace).unwrap(); let initial = backend @@ -1019,7 +1019,7 @@ mod tests { // Seed domain handling evidence under a transaction. This fixture // exercises owner purge, not singleton authentication or delivery. store - .with_immediate_transaction(|tx| { + .with_transaction(|tx| { ticket::sqlite_update_ticket_assignment_tracking_in_transaction( tx, workspace, diff --git a/crates/workspace-server/src/workspace_drive_host.rs b/crates/workspace-server/src/workspace_drive_host.rs index bc0bfc390..410c0c049 100644 --- a/crates/workspace-server/src/workspace_drive_host.rs +++ b/crates/workspace-server/src/workspace_drive_host.rs @@ -58,13 +58,10 @@ pub(crate) struct WorkspaceDriveHost { impl WorkspaceDriveHost { pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result { let storage = storage.clone(); - let registration = crate::db_executor::DbExecutor::shared() - .run(move || { - Drive::register(&storage).map_err(|error| { - Error::Store(format!("failed to register Drive storage: {error}")) - }) - }) - .await?; + let registration = tokio::task::spawn_blocking(move || Drive::register(&storage)) + .await + .map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))? + .map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?; Ok(Self { registration, purged: AtomicBool::new(false), @@ -83,21 +80,21 @@ impl WorkspaceDriveHost { } let storage = storage.clone(); let registration = self.registration.clone(); - let drive = crate::db_executor::DbExecutor::shared() - .run(move || { - reject_symlink_ancestors(&root)?; - // Domain IMMEDIATE transactions lock the attached server authority - // as well as Drive metadata. Deletion reservation and cached-handle - // operations therefore serialize with other server DB writes. - Drive::open_with_workspace_authority( - &storage, - ®istration, - &root, - &authority_database, - ) - .map_err(Error::from) - }) - .await?; + let drive = tokio::task::spawn_blocking(move || { + reject_symlink_ancestors(&root)?; + // Domain IMMEDIATE transactions lock the attached server authority + // as well as Drive metadata. Deletion reservation and cached-handle + // operations therefore serialize with other server DB writes. + Drive::open_with_workspace_authority( + &storage, + ®istration, + &root, + &authority_database, + ) + .map_err(Error::from) + }) + .await + .map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??; *cached = Some(drive.clone()); Ok(drive) } @@ -112,9 +109,9 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root, authority_database).await?; - crate::db_executor::DbExecutor::shared() - .run(move || drive.fence().map_err(Error::from)) + tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from)) .await + .map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))? } pub(crate) async fn purge( @@ -127,20 +124,20 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root.clone(), authority_database).await?; - crate::db_executor::DbExecutor::shared() - .run(move || -> Result<()> { - reject_symlink_ancestors(&root)?; - // Domain purge persists its tombstone before removing blobs/nodes/receipts. - drive.purge()?; - // Remove the empty blob root, never the data root or another Workspace. - // Missing roots are successful retries; other failures retain FeatureStorage. - match std::fs::remove_dir_all(&root) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - } - }) - .await?; + tokio::task::spawn_blocking(move || -> Result<()> { + reject_symlink_ancestors(&root)?; + // Domain purge persists its tombstone before removing blobs/nodes/receipts. + drive.purge()?; + // Remove the empty blob root, never the data root or another Workspace. + // Missing roots are successful retries; other failures retain FeatureStorage. + match std::fs::remove_dir_all(&root) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + }) + .await + .map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??; // A later signing-material/finalization failure can retry after the // feature DB has been closed and removed by FeatureStorage.delete. self.purged.store(true, Ordering::Release); diff --git a/crates/workspace-server/src/workspace_signing_identity.rs b/crates/workspace-server/src/workspace_signing_identity.rs index 959f28805..f724f2ca1 100644 --- a/crates/workspace-server/src/workspace_signing_identity.rs +++ b/crates/workspace-server/src/workspace_signing_identity.rs @@ -780,7 +780,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("server.db"); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); store .upsert_account(&AccountRecord { account_id: "account-1".to_string(), @@ -833,7 +833,7 @@ mod tests { drop(failing_service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); let service = WorkspaceSigningIdentityService::new(store.clone(), materials.clone()); let provisioned = service .provision_existing("workspace-1", "account-1") @@ -962,7 +962,7 @@ mod tests { ); drop(service); drop(store); - let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap()); + let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap()); let restarted = WorkspaceSigningIdentityService::new(store.clone(), materials.clone()); let recovered = restarted .provision_existing("workspace-2", "account-1") diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index a62faa34e..8b0e0b818 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -117,13 +117,11 @@ pub(crate) async fn serve_workspace_subscription( worker_id, runtime_id: Some(runtime_id), } => { - let connection = match WorkspaceWorker::resolve_async( + let connection = match WorkspaceWorker::resolve( &api, &runtime_id, worker_id.as_str(), - ) - .await - { + ) { Ok(worker) => worker .connect_protocol(&context) .await @@ -368,8 +366,7 @@ async fn run_workspace_workers( ) { let Ok(mut projection) = api .worker_projection - .subscribe_ordered_async(api.config.max_records.max(1)) - .await + .subscribe_ordered(api.config.max_records.max(1)) else { let _ = send_rejected( &outbound, @@ -380,22 +377,11 @@ async fn run_workspace_workers( .await; return; }; - let Ok(mut workers) = project_registry_workers_async( - api.store.clone(), - api.config.workspace_id.clone(), - projection.take_snapshot(), - ) - .await - else { - let _ = send_rejected( - &outbound, - request_id, - SubscriptionRejectionCode::Internal, - "failed to enrich durable Worker projection".to_string(), - ) - .await; - return; - }; + let mut workers = projection + .take_snapshot() + .into_iter() + .filter_map(|record| project_registry_worker(&api, record)) + .collect::>(); sort_workers(&mut workers); if send_frame( &outbound, @@ -420,29 +406,7 @@ async fn run_workspace_workers( for change in event.changes { let payload = match change { crate::worker_projection::WorkerProjectionChange::Upsert(record) => { - let workers = project_registry_workers_async( - api.store.clone(), - api.config.workspace_id.clone(), - vec![record], - ) - .await; - let Ok(mut workers) = workers else { - // Admission failure cannot silently lose a live - // update: terminate and require a fresh snapshot. - let _ = send_frame( - &outbound, - SubscriptionFrame::new(SubscriptionFramePayload::Event( - SubscriptionEvent::SubscriptionClosed { - subscription_id: subscription_id.clone(), - code: SubscriptionTerminationCode::Lagged, - message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(), - }, - )), - ) - .await; - return; - }; - let Some(worker) = workers.pop() else { + let Some(worker) = project_registry_worker(&api, record) else { continue; }; SubscriptionEventPayload::WorkerUpserted { worker } @@ -486,28 +450,8 @@ async fn run_workspace_workers( } } -async fn project_registry_workers_async( - store: std::sync::Arc, - workspace_id: String, - records: Vec, -) -> crate::Result> { - store - .db_executor() - .run(move || { - // Keep the committed Worker payload; only the existing Workspace-scoped - // workdir/history/repository enrichment reads occur inside this job. - Ok(records - .into_iter() - .filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record)) - .collect()) - }) - .await -} - -// Synchronous helpers stay synchronous; they run only inside the job above. fn project_registry_worker( - store: &dyn crate::store::ControlPlaneStore, - workspace_id: &str, + api: &WorkspaceApi, record: crate::store::WorkerRegistryProjectionRecord, ) -> Option { let runtime_id = record.registry.worker.runtime_id.clone(); @@ -547,25 +491,29 @@ fn project_registry_worker( attempt_id: job.attempt_id, purpose: job.purpose, }); - if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) { + if !project_working_directory(api, &record.registry.worker, &mut worker) { return None; } Some(worker) } fn project_working_directory( - store: &dyn crate::store::ControlPlaneStore, - workspace_id: &str, + api: &WorkspaceApi, worker_ref: &RuntimeWorkerRef, worker: &mut SubscriptionWorker, ) -> bool { - let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else { + let Ok(links) = api + .store + .list_worker_workdir_links(&api.config.workspace_id, worker_ref) + else { return false; }; if !links.is_empty() { let mut attachments = Vec::with_capacity(links.len()); for link in links { - let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id) + let Ok(Some(workdir)) = api + .store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) else { return false; }; @@ -585,8 +533,9 @@ fn project_working_directory( } worker.workdir_attachments = attachments; } else { - let Ok(has_link_history) = - store.worker_workdir_link_history_exists(workspace_id, worker_ref) + let Ok(has_link_history) = api + .store + .worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref) else { return false; }; @@ -594,17 +543,13 @@ fn project_working_directory( worker.workdir_attachments.clear(); } } - project_repository_keys(store, workspace_id, worker) + project_repository_keys(api, worker) } -fn project_repository_keys( - store: &dyn crate::store::ControlPlaneStore, - workspace_id: &str, - worker: &mut SubscriptionWorker, -) -> bool { +fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool { project_repository_keys_with(worker, |repository_id| { - store - .get_repository(workspace_id, repository_id) + api.store + .get_repository(&api.config.workspace_id, repository_id) .ok() .flatten() .map(|repository| repository.repository_key) @@ -689,8 +634,9 @@ mod tests { protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap() } - fn external_worker() -> SubscriptionWorker { - SubscriptionWorker { + #[test] + fn external_workdir_attachment_does_not_remove_workspace_worker() { + let mut worker = SubscriptionWorker { restore_observation_token: None, worker_id: worker_id(), runtime_id: Some("runtime-1".to_string()), @@ -714,12 +660,8 @@ mod tests { .unwrap(), }, ], - } - } + }; - #[test] - fn external_workdir_attachment_does_not_remove_workspace_worker() { - let mut worker = external_worker(); assert!(project_repository_keys_with(&mut worker, |_| { panic!("External Workdir must not require Repository projection") })); @@ -731,110 +673,6 @@ mod tests { ); } - #[tokio::test(flavor = "current_thread")] - async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository() - { - use crate::store::{ - AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord, - WorkspaceRecord, - }; - let store = std::sync::Arc::new( - SqliteWorkspaceStore::in_memory_for_test() - .unwrap() - .with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()), - ); - store - .upsert_account(&AccountRecord { - account_id: "owner".into(), - kind: "user".into(), - handle: "owner".into(), - display_name: "Owner".into(), - created_at: "1".into(), - updated_at: "1".into(), - }) - .unwrap(); - store - .upsert_workspace(&WorkspaceRecord { - workspace_id: "workspace-1".into(), - owner_account_id: "owner".into(), - display_name: "Enrichment".into(), - state: "active".into(), - created_at: "1".into(), - updated_at: "1".into(), - }) - .await - .unwrap(); - let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1"); - store - .upsert_worker_registry(&WorkerRegistryRecord { - workspace_id: "workspace-1".into(), - worker: worker_ref.clone(), - display_name: "Durable name".into(), - profile: Some("builtin:coder".into()), - retention_state: "normal".into(), - transcript_ref: None, - session_ref: None, - summary_ref: None, - diagnostics_ref: None, - created_at: "1".into(), - updated_at: "1".into(), - }) - .unwrap(); - store - .apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2") - .unwrap(); - let committed = store - .worker_registry_projection("workspace-1", &worker_ref) - .unwrap() - .unwrap(); - // Change authority after capturing the payload, as can happen while async - // enrichment is queued. It must not re-query and replace the committed Worker. - let mut later = external_worker(); - later.state = protocol::subscription::SubscriptionWorkerState::Running; - store - .apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3") - .unwrap(); - let projected = project_registry_workers_async( - store.clone(), - "workspace-1".into(), - vec![committed.clone()], - ) - .await - .unwrap(); - assert_eq!(projected.len(), 1); - let worker = &projected[0]; - assert_eq!( - worker.state, - protocol::subscription::SubscriptionWorkerState::Idle - ); - assert_eq!( - worker.availability, - protocol::subscription::SubscriptionWorkerAvailability::Observed - ); - assert_eq!(worker.display_name.as_deref(), Some("Durable name")); - assert_eq!(worker.profile.as_deref(), Some("builtin:coder")); - assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1")); - assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1")); - assert_eq!( - worker.workdir_attachments, - external_worker().workdir_attachments - ); - let mut unresolved = committed; - unresolved - .observation - .as_mut() - .unwrap() - .worker - .workdir_attachments[0] - .repository_id = Some("missing-repository".into()); - assert!( - project_registry_workers_async(store, "workspace-1".into(), vec![unresolved]) - .await - .unwrap() - .is_empty() - ); - } - async fn run_with_events(events: mpsc::Receiver) -> SubscriptionResponse { let (control_outbound, mut control_receiver) = mpsc::channel(1); let (protocol_outbound, _protocol_receiver) = mpsc::channel(1); diff --git a/docs/development/backend-db-execution.md b/docs/development/backend-db-execution.md deleted file mode 100644 index 8dd00b478..000000000 --- a/docs/development/backend-db-execution.md +++ /dev/null @@ -1,115 +0,0 @@ -# Backend bounded synchronous execution - -## Ownership and admission - -`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner. -Use `executor.run(move || synchronous_operation(...)).await` with owned input and -services. The closure owns the **whole** synchronous operation, including connection -acquisition, mutex wait, the existing SQL transaction, commit, and ordered local -publication. Do not split source authorization, receipts, CAS, audit, or saved -results around an await. Do not change connection ownership or introduce a read pool. - -Production stores share one DB worker and a queue of 32 jobs. Admission uses a -nonblocking queue send and a separate actual-job budget (worker count plus queue -capacity); rejection means the operation was **not accepted**. HTTP maps capacity -and executor unavailability to 503, not authentication failure or deletion conflict. -Tests can inject a dedicated executor with explicit limits, without process-global -settings or environment variables. Ordinary SQLite fixtures use `open_for_test`, -`in_memory_for_test`, or `from_connection_for_test`: their clones share their own -1-worker/32-queued owner, not the production singleton or another fixture's drain. -Server/Workspace fixture constructors likewise inject a lazy `HostExecutors` owner; -all child Workspaces and weak retention adapters use that same owner. Host and -cleanup still have the production 2-worker/16-queued limits. Dropping the final -sender closes its queue after accepted work finishes; idle fixture threads are -not kept in a process-global fixture cache. - -The restore intent journal and Workspace deletion use -`with_immediate_transaction`: read authority/journal state only after acquiring -SQLite write ownership. A DEFERRED read-to-write upgrade can return BUSY without -invoking SQLite's busy handler when another connection is writing. Moving these -whole helpers to an offloaded caller must not create an automatic mutation retry. - - -Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter: - -- does not interrupt, resubmit, or retry SQL; -- does not return its queue/job slot before execution and discarded-result cleanup; -- does not release the request lifecycle lease before actual completion. - -Synchronous post-commit continuations inherit the admitted lifecycle source when -submitting another bounded job. This is not implicit inheritance by detached Tokio -tasks. An operation panic has an unknown outcome even if SQL already committed; -return a structured unknown-outcome error and diagnostic, never retry the mutation. -A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all -jobs accepted before its barrier is first polled and waits for actual cleanup; it -is not a queue close and does not include later admissions. Lifecycle callers must -stop their producers/admission before destroying resources. - -## Runtime/network boundaries - -A synchronous Runtime workflow must not monopolize the sole DB worker while -waiting for callbacks that themselves need DB authority. The Server's separate -finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job -ownership mechanism, not an unbounded `spawn_blocking` queue. - -Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can -use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an -owned workflow needs async resource locks, `run_host_workflow` polls it on a Host -worker using the original Tokio handle; do not build a replacement runtime for -cached services and do not submit to and await the Host lane from inside itself. -Control/session guards needed by an accepted mutation belong to that actual job. - -Backend Job cleanup has a third finite lane (two workers, 16 queued jobs), so a -long Run teardown wait cannot starve ordinary Host control/stop requests. It is -admitted before its durable cleanup claim. Rejection leaves Pending/Failed -ownership recoverable instead of abandoning an Executing claim. Ticket -checker dispatch and timeout processing use bounded admission; overload leaves the -saved reservation/deadline diagnosable. They are not automatically retried after an -unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof -of model processing. - -Remote subscription and Workdir authorization are async callbacks: their complete -live binding check and fresh request signing are offloaded together. A queue error -must not fall back to a configured bearer token. Synchronous CLI/migration callers -and existing synchronous Runtime adapters retain their explicit blocking boundary. - -## Caller recipe (including assignment APIs) - -```rust -let store = api.store.clone(); -let workspace_id = api.workspace_id().to_owned(); -let worker = worker.clone(); -let record = api.store.db_executor().run(move || { - store.get_worker_registry(&workspace_id, &worker) -}).await?; -``` - -This is an illustration, not a new authority API. Assignment delivery scan, -reserve, prepare, acknowledgement and start authorization must keep their existing -whole transaction helpers. Additional assignment diagnostics, answer mutations, -and Ticket queries use the same owned caller boundary; do not add a second ledger -or reconstruct processing evidence from Worker liveness. - -## Regression evidence - -- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread - timer/control task progresses; accepted/cancelled queue bounds; rollback; panic - before/after commit; discarded reply cleanup and retained authority; child jobs. -- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope - opening/mutation under a separate SQLite connection's write lock; overload; - cancelled HTTP jobs and Workspace deletion drain; same-budget fixture isolation, - saturated/cancelled Host jobs versus cleanup and paused Tokio time; restore intent - contention before the first authority read, with one persisted tuple and no retry. - The terminal checker fixture keeps its original Tokio driver alive through accepted cleanup, - then shuts down projection producers and drains actual DB jobs. -- `worker_projection::tests`: real SQL contention, commit/payload publication, - ordered snapshot registration, and replacement draining a cancelled consumer's - accepted SQL before applying a new Runtime snapshot. -- Auth/proof and Runtime subscription tests protect availability classification, - replay rejection, and no bearer fallback. -- `workdir`'s `http-client` tests protect awaited fresh authorization and exact - request claims before any HTTP send. - -These fixtures use temporary authority data and observable synchronization, not -live databases or a fixed sleep to guess completion. This change does not claim a -latency benchmark or a measured production incident. diff --git a/docs/report/2026-10-12-t748-suite-diagnosis.md b/docs/report/2026-10-12-t748-suite-diagnosis.md deleted file mode 100644 index 3e73e67c2..000000000 --- a/docs/report/2026-10-12-t748-suite-diagnosis.md +++ /dev/null @@ -1,166 +0,0 @@ -# T-748: default-parallel suite failure diagnosis - -## Sources and preserved observations - -- Original local candidate: `61533fadb6b39465af60cb55aed2b4d0650319cd`. -- Comparison source: `56795fb92cb9def9bcd701bd51d92e99526f8eff` (develop/T742). -- Same implementation branch: `work/T-748-bounded-db-execution`. -- The original `target/t748-server-tests.log` remains untouched. Its 97 FAILED - lines and 600-second timeout were symptoms, not proof of one cause. -- Initial diagnosis used `cargo test -p yoi-workspace-server -- --nocapture`, - default libtest parallelism, without changing queue limits, features, or startup - settings. `target/t748-diagnostic-environment.txt` records the environment; - this host reports 24 available CPUs. Diagnostic captures were bounded to - 90/150 seconds to obtain errors before the original timeout, not counted as - successful full-suite runs. - -Logs under `target/` are local diagnostic artifacts, not tracked review inputs. -This report records their meaning and the permanent regressions that reproduce it. - -## 1. Different fixtures consumed one global Host admission budget - -The first non-intentional errors in `target/t748-server-nocapture.log` were -`DbCapacityExceeded` in manual Worker setup (`server.rs` at the then-current -line 50614), followed by assignment fixture construction (52990/53045) and -HTTP 503 responses. The deliberate before/after-commit executor panic tests -are different: those passed and verified unknown-outcome diagnostics. - -Temporary submit/start/cleanup instrumentation recorded submitting test names, -closure types, actual job IDs and the execution-owner identity. It showed a -single Host lane with `max=18`, with accepted jobs from unrelated fixtures. -A rejection snapshot contained `{32,45,46,...,61}`. The complete diagnostic -capture later contained cleanup for every submitted Host job on that owner; -it did not show permanently abandoned accepted Host jobs. Instrumentation was -removed before the source was committed/published. - -`target/t748-live-process-18.txt` sampled the test process and threads at the -actual rejection. That sample had 126 threads; the Host workers were runnable, -and the process was using approximately 17 CPU cores. There was no captured -thread-creation EAGAIN/unavailable error. This evidence does not establish an -OS thread/CPU hard limit as the cause. No test processes remained after the -bounded diagnostic run or the completed suite. - -A representative failure passed alone (0.56 seconds); the eight logging cases -passed with four test threads (2.46 seconds). The isolated comparison source's -full default-parallel suite passed: 931 library tests, one pre-existing ignored -migration fixture, 11 binary tests; library duration 168.74 seconds. This did -not prove that all new failures were capacity-related, but did distinguish the -new shared-admission rejection from an observed baseline failure. - -### Ownership fix (`f3d4aba5`) - -Production budgets remain shared and finite: DB 1 worker/32 queued, Host 2/16, -cleanup 2/16. Ordinary fixtures now explicitly own a finite DB executor and lazy -Host/cleanup owners at those same limits. API clones, child Workspace APIs, and -weak retention adapters use their injected owner. No global fixture lane cache, -parallelism reduction, unlimited queue, or capacity-wait retry was added. - -The permanent saturated-Host regression fills both running slots and all 16 -queued slots, cancels waiters, advances paused Tokio time, and still observes -rejection until actual completion. Another fixture and the separate cleanup lane -can progress; draining restores the original budget and completes exactly 18 -accepted jobs. Existing real-SQLite cancellation/lease/drain tests remain active. - -## 2. SQL offload invalidated fixture assumptions about first poll and publication - -After Host isolation the default-parallel suite finished in 171.57 seconds: -954 passed, three failed, one existing ignored. Actual errors were: - -- removal subscriber saw a legitimate queued Runtime stop Upsert before Removed; -- external session-open expiry exceeded its one-second bound while its DB lane - was still shared across other fixtures; -- grant replacement returned success because one initial poll had only submitted - the offloaded authority read, not reached the held original-grant lock. - -The ordinary DB fixtures were isolated too. The one-second expiry assertion was -not increased. Grant testing now drains its actual authority read then polls the -saved response to reach the held lock before replacing the grant. Removal testing -observes the committed stopped projection before taking the removal baseline. -The Removed payload assertion is unchanged. These are observable synchronization, -not sleeps or retries. Focused expiry, grant, and removal tests all passed. - -A captured cleanup job also attempted to poll a Tokio timer after its fixture -runtime had shut down. The terminal checker fixture now retains that original -runtime through accepted Host and cleanup jobs, then stops projection producers -and drains actual DB work before ending. Terminal SQL state is not treated as -actual cleanup completion. The subsequent full run contained only the three -intentional executor panic diagnostics, not the shutdown-context panic. - -The synchronization/teardown changes are in `df9d056d`. - -## 3. Restore intent's DEFERRED read-to-write upgrade returned SQLite BUSY - -The next default-parallel run finished in 171.48 seconds with 956 passed, two -failed, one existing ignored. Both failures were SQLite `DatabaseBusy`, extended -code 5, not capacity rejection: - -- production-profile zero-input Orchestrator restore; -- pending manual Worker claim restore reconciliation (500 instead of 409). - -Both failed alone too. `target/t748-busy-origin.log` captured the error and a -backtrace through `pin_internal_worker_restore_intent` -> the transaction helper --> internal restore -> manual claim. The journal read authority first in a -DEFERRED transaction while the offloaded projection's other SQLite connection -could write. Such an upgrade can return BUSY without invoking SQLite's busy -handler; treating this as a retryable whole mutation would be incorrect. - -A new real-SQLite regression holds an IMMEDIATE writer on another connection, -registers an owned per-connection busy observer, and attempts one journal pin. -The old helper fails deterministically with BUSY/5 before waiting for ownership -(`target/t748-deferred-upgrade-red.log`, 0.23 seconds). The fixed helper waits -before its first authority read, while a current-thread timer and control task -progress. After releasing the writer it persists exactly one requested tuple; -there is no application-level mutation retry. - -`f597f1b5` makes this existing mutation helper explicitly -`with_immediate_transaction`. Restore journal and deletion authority reads, -mutation and commit remain in one transaction. Focused DB/HTTP tests (8), the -original restore and claim cases, and deletion tests (13) passed. - -## 4. Why a pre-dispatch failure could become the 600-second suite timeout - -The original long-running logging case joined an HTTP request with a fake provider -waiting for command dispatch. On pre-dispatch rejection the HTTP future can finish -with 503 while the receiver remains Pending: its channel is still owned by the -fixture, but no command will arrive. An unbounded `join!` waits forever in that -state and hides the actual error behind the outer suite timeout. - -The permanent `saturated_host_before_command_start_returns_availability_without_a_provider_operation` -regression uses the real signed HTTP route with a controlled full Host budget. It -observes the final 503/`db_capacity_exceeded` response and the still-live Pending -provider receiver, with no process or command dispatch. The original logging -fixture now asserts an early HTTP response before waiting for both branches, so -an unexpected rejection reports its body immediately rather than hanging. This -reproduces the causal timeout mechanism; the old log itself has no stack trace -from the hung provider future. - -The nine logging cases pass together at default parallelism (`49bfbfb2`). - -## Artifact/source isolation during comparison - -Initially the archived baseline used the existing Cargo target directory. That -replaced a same-package artifact/fingerprint; one later post-fix representative -command ran the baseline's 932-test binary (and `db_executor --list` had no tests). -That result was rejected as candidate evidence. All later candidate checks/tests -use `--target-dir target/t748-candidate-build`, separate from the baseline cache. -This changes artifact location, not test parallelism, features or admission policy. -The fresh candidate build listed the expected executor tests and total test count. - -## Validation and remaining composition boundary - -Final code source `49bfbfb2` passed the full default-parallel Server suite, -including the command-overload case: 960 passed, zero failures, one existing -ignored migration fixture; 11 binary tests, library duration 175.29 seconds. -The only captured panics were the three deliberate unknown-outcome tests. -Root `cargo check` compile closure passed (including TUI/Server), as did Workdir -default (153 tests) and `http-client` (156 tests) full suites, format/diff checks -and state-contract names (558 matches, zero unapproved, zero stale exceptions). -These commands used the isolated candidate artifact directory; libtest default -parallelism and the production budgets were unchanged. The MR records the exact -published source, which also includes this report and the boundary document. - -No latency benchmark, production incident measurement, target integration, -Ticket conclusion, or running-environment update is claimed. T743 has now -published its caller inventory/source; its owner will compose the T748 facade and -new callers, then obtain review of that exact combined source. T748 does not copy -those domain APIs or wait on the former unpublished branch.