From b1076f1d6530e8ee7d9023408417107f1f141113 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 12 Oct 2026 06:43:16 +0900 Subject: [PATCH] feat(backend): bound synchronous database and host execution ownership --- crates/workdir/src/http.rs | 161 +- crates/workspace-server/src/auth.rs | 202 +- crates/workspace-server/src/db_executor.rs | 957 ++ .../src/frozen_content_state_tests.rs | 2 + .../src/frozen_repository_key_tests.rs | 4 + .../src/frozen_runtime_binding_tests.rs | 2 + crates/workspace-server/src/hosts.rs | 145 +- crates/workspace-server/src/lib.rs | 7 + .../src/runtime_subscription.rs | 13 +- crates/workspace-server/src/server.rs | 7916 ++++++++++------- .../src/server/subjektiv_job.rs | 146 +- .../src/server/tests/auth_logging_tests.rs | 2 +- .../tests/protocol_backpressure_tests.rs | 5 +- .../server/tests/workspace_admission_tests.rs | 58 +- .../src/server/worker_operations.rs | 349 +- .../src/server/workspace_admission.rs | 22 + .../src/server_db_execution_tests.rs | 606 ++ crates/workspace-server/src/server_drive.rs | 13 +- .../src/server_drive_grants.rs | 73 +- .../src/server_workspace_config.rs | 431 +- .../src/server_workspace_config_tests.rs | 4 +- crates/workspace-server/src/store.rs | 679 +- .../workspace-server/src/worker_projection.rs | 455 +- crates/workspace-server/src/worker_source.rs | 42 +- .../src/worker_source/runtime_request.rs | 410 +- .../src/workspace_drive_host.rs | 73 +- .../src/workspace_subscription.rs | 221 +- docs/development/backend-db-execution.md | 95 + 28 files changed, 9192 insertions(+), 3901 deletions(-) create mode 100644 crates/workspace-server/src/db_executor.rs create mode 100644 crates/workspace-server/src/server_db_execution_tests.rs create mode 100644 docs/development/backend-db-execution.md diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index 940506ca7..ae155b008 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -432,16 +432,23 @@ mod client { use super::*; use crate::{Workdir, WorkdirSession}; - /// Provides a fresh bearer token for each Runtime request. Backend - /// implementations can mint short-lived capability tokens without making a - /// Worker-bound session expire with the token used to open it. + /// An owned, Send authorization future that can offload synchronous authority + /// work without borrowing request data or the authorization provider. + pub type WorkdirHttpAuthorizationFuture = std::pin::Pin< + Box> + Send + 'static>, + >; + + /// Provides a fresh bearer token asynchronously for each Runtime request. + /// Backend implementations can mint short-lived capability tokens without + /// making a Worker-bound session expire with the token used to open it. + /// Blocking DB/signing work must be offloaded by the implementation. pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync { fn bearer_token( &self, - method: &str, - path_and_query: &str, - body: &[u8], - ) -> Result; + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture; } struct FixedBearerToken(Arc); @@ -455,11 +462,12 @@ mod client { impl WorkdirHttpAuthorization for FixedBearerToken { fn bearer_token( &self, - _method: &str, - _path_and_query: &str, - _body: &[u8], - ) -> Result { - Ok(self.0.to_string()) + _method: String, + _path_and_query: String, + _body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let token = self.0.clone(); + Box::pin(async move { Ok(token.to_string()) }) } } @@ -509,7 +517,9 @@ mod client { )?; let body = serde_json::to_vec(&request) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = authorization.bearer_token("POST", url.path(), &body)?; + let token = authorization + .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) + .await?; let response = client .post(url) .bearer_auth(token) @@ -560,7 +570,10 @@ mod client { let operation = WorkdirSessionOperationRequest { operation }; let body = serde_json::to_vec(&operation) .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; - let token = self.authorization.bearer_token("POST", url.path(), &body)?; + let token = self + .authorization + .bearer_token("POST".to_string(), url.path().to_string(), body.clone()) + .await?; let response = self .client .post(url) @@ -864,7 +877,10 @@ mod client { &self.base_url, &["v1", "workdir-sessions", self.session_id.as_str()], )?; - let token = self.authorization.bearer_token("DELETE", url.path(), &[])?; + let token = self + .authorization + .bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new()) + .await?; let response = self .client .delete(url) @@ -918,11 +934,124 @@ mod client { WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string()) } + #[cfg(test)] + mod authorization_tests { + use super::*; + use std::sync::Mutex; + + #[derive(Debug, Default)] + struct RejectAuthorization { + requests: Arc)>>>, + } + + impl WorkdirHttpAuthorization for RejectAuthorization { + fn bearer_token( + &self, + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let requests = self.requests.clone(); + Box::pin(async move { + tokio::task::yield_now().await; + let mut requests = requests.lock().unwrap(); + requests.push((method, path_and_query, body)); + Err(WorkdirError::Unavailable(format!( + "authorization rejected request {}", + requests.len() + ))) + }) + } + } + + #[tokio::test] + async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let base_url: Url = format!("http://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(); + let client = Client::builder().no_proxy().build().unwrap(); + let authorization = Arc::new(RejectAuthorization::default()); + let workdir = Workdir::new("w"); + let open_request = OpenWorkdirSessionRequest { + owner_worker_id: Some("worker".to_string()), + }; + let error = RemoteWorkdirSession::open_with_authorization( + client.clone(), + base_url.clone(), + authorization.clone(), + workdir.id().clone(), + open_request.clone(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 1" + )); + + // Seed an already-open session to exercise the public operation and + // close boundaries independently of the rejected open request. + let remote = RemoteWorkdirSession { + client, + base_url, + authorization: authorization.clone(), + workdir, + session_id: WorkdirSessionId::new("s").unwrap(), + capabilities: WorkdirSessionCapabilities::EMPTY, + closed: AtomicBool::new(false), + }; + let stat = StatRequest { + path: crate::WorkdirPath::new("file").unwrap(), + }; + let error = remote.stat(stat.clone()).await.unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 2" + )); + let error = remote.close().await.unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == "authorization rejected request 3" + )); + assert_eq!( + *authorization.requests.lock().unwrap(), + vec![ + ( + "POST".to_string(), + "/v1/working-directories/w/sessions".to_string(), + serde_json::to_vec(&open_request).unwrap(), + ), + ( + "POST".to_string(), + "/v1/workdir-sessions/s/operations".to_string(), + serde_json::to_vec(&WorkdirSessionOperationRequest { + operation: WorkdirSessionOperation::Stat(stat), + }) + .unwrap(), + ), + ( + "DELETE".to_string(), + "/v1/workdir-sessions/s".to_string(), + Vec::new(), + ), + ] + ); + assert_eq!( + listener.accept().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); + } + } + pub use self::RemoteWorkdirSession as ClientSession; } #[cfg(feature = "http-client")] -pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization}; +pub use client::{ + ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture, +}; #[cfg(test)] mod tests { diff --git a/crates/workspace-server/src/auth.rs b/crates/workspace-server/src/auth.rs index aa202f2de..bfe4ab4c3 100644 --- a/crates/workspace-server/src/auth.rs +++ b/crates/workspace-server/src/auth.rs @@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option { }) } -pub async fn resolve_request_actor( - store: &S, +pub async fn resolve_request_actor( + store: std::sync::Arc, headers: &HeaderMap, cookie_name: &str, ) -> Result> { - if let Some(token) = parse_bearer(headers) { + let bearer = parse_bearer(headers); + let session = parse_cookie(headers, cookie_name); + if bearer.is_none() && session.is_none() { + return Ok(None); + } + let executor = store.db_executor(); + executor + .run(move || resolve_request_actor_sync(store.as_ref(), bearer, session)) + .await +} + +// One owned synchronous stage includes token usage marking and the user read. +// Do not call async store wrappers here: they use the same executor themselves. +fn resolve_request_actor_sync( + store: &dyn ControlPlaneStore, + bearer: Option, + session: Option, +) -> Result> { + if let Some(token) = bearer { let token_hash = token_hash(&token); if let Some(api_token) = store.resolve_api_token(&token_hash)? { if api_token @@ -109,7 +127,7 @@ pub async fn resolve_request_actor( } } - if let Some(session_token) = parse_cookie(headers, cookie_name) { + if let Some(session_token) = session { if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? { if is_expired(&session.expires_at) { return Ok(None); @@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error { #[cfg(test)] mod tests { use super::*; + use crate::store::{ + AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord, + }; + use std::sync::Arc; + + fn seeded_store() -> Arc { + let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::in_memory() + .unwrap() + .with_db_executor(executor), + ); + store + .upsert_account(&AccountRecord { + account_id: "account-auth".into(), + kind: "user".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .upsert_user(&UserRecord { + user_id: "user-auth".into(), + account_id: "account-auth".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .create_browser_session(&BrowserSessionRecord { + session_id: "session-auth".into(), + token_hash: token_hash("session-secret"), + user_id: "user-auth".into(), + created_at: "1".into(), + expires_at: "9999-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + store + } + + #[tokio::test] + async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() { + let store = seeded_store(); + let hash = token_hash("api-secret"); + store + .create_api_token(&ApiTokenRecord { + token_id: "token-auth".into(), + token_hash: hash.clone(), + user_id: "user-auth".into(), + label: "Auth".into(), + created_at: "1".into(), + expires_at: None, + revoked_at: None, + last_used_at: None, + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap()); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let actor = resolve_request_actor(store.clone(), &headers, "session") + .await + .unwrap() + .unwrap(); + assert_eq!(actor.user_id, "user-auth"); + assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken); + assert!( + store + .resolve_api_token(&hash) + .unwrap() + .unwrap() + .last_used_at + .is_some() + ); + } + + #[tokio::test] + async fn expired_api_token_does_not_fall_back_to_valid_browser_session() { + let store = seeded_store(); + let hash = token_hash("expired-secret"); + store + .create_api_token(&ApiTokenRecord { + token_id: "token-auth".into(), + token_hash: hash.clone(), + user_id: "user-auth".into(), + label: "Auth".into(), + created_at: "1".into(), + expires_at: Some("2000-01-01T00:00:00Z".into()), + revoked_at: None, + last_used_at: None, + }) + .unwrap(); + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + "Bearer expired-secret".parse().unwrap(), + ); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + assert!( + resolve_request_actor(store.clone(), &headers, "session") + .await + .unwrap() + .is_none() + ); + assert!( + store + .resolve_api_token(&hash) + .unwrap() + .unwrap() + .last_used_at + .is_none() + ); + } + + #[tokio::test] + async fn unknown_bearer_can_fall_back_to_valid_browser_session() { + let store = seeded_store(); + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + "Bearer unknown-secret".parse().unwrap(), + ); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let actor = resolve_request_actor(store, &headers, "session") + .await + .unwrap() + .unwrap(); + assert_eq!(actor.user_id, "user-auth"); + assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession); + } + + #[tokio::test] + async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() { + let store = seeded_store(); + let executor = store.db_executor(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let blocker_executor = executor.clone(); + let blocker = tokio::spawn(async move { + blocker_executor + .run(move || -> Result<()> { + let _ = started.send(()); + release_rx.recv().unwrap(); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(1), started_rx) + .await + .unwrap() + .unwrap(); + let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) })); + assert!(futures::poll!(queued.as_mut()).is_pending()); + let mut headers = HeaderMap::new(); + headers.insert(header::COOKIE, "session=session-secret".parse().unwrap()); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + resolve_request_actor(store, &headers, "session"), + ) + .await; + release.send(()).unwrap(); + blocker.await.unwrap().unwrap(); + queued.await.unwrap(); + assert!(matches!( + result.expect("capacity rejection must not wait for queued SQL"), + Err(Error::DbCapacityExceeded) + )); + } #[test] fn user_code_uses_the_uuid_random_tail() { diff --git a/crates/workspace-server/src/db_executor.rs b/crates/workspace-server/src/db_executor.rs new file mode 100644 index 000000000..1c44ef8c9 --- /dev/null +++ b/crates/workspace-server/src/db_executor.rs @@ -0,0 +1,957 @@ +//! Bounded execution of whole synchronous DB operations, independent of Tokio. +//! +//! Accepted jobs belong to the workers, not their async waiters. Cancelling a +//! waiter does not cancel or retry SQL, release capacity, or release its lease. +use std::{ + collections::BTreeSet, + future::Future, + num::NonZeroUsize, + panic::{AssertUnwindSafe, catch_unwind}, + sync::{Arc, Mutex, OnceLock, mpsc}, +}; +use tokio::sync::{Notify, oneshot}; + +use crate::{Error, Result}; + +/// The default is one FIFO worker and at most 32 waiting operations. +#[derive(Clone, Copy, Debug)] +pub struct DbExecutorLimits { + pub workers: NonZeroUsize, + pub queued: NonZeroUsize, +} + +impl Default for DbExecutorLimits { + fn default() -> Self { + Self { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(32).unwrap(), + } + } +} + +/// A request can retain its lifecycle authority until actual SQL finishes. +/// Retention happens at submission, while the caller's task-local scope exists. +pub(crate) trait DbOperationLifetime: Send + Sync { + fn retain(&self) -> Box; +} + +tokio::task_local! { + static OPERATION_LIFETIME: Arc; +} + +pub(crate) async fn with_operation_lifetime( + lifetime: Arc, + future: F, +) -> F::Output { + OPERATION_LIFETIME.scope(lifetime, future).await +} + +#[derive(Default)] +struct Accepted { + last_job_id: u64, + unfinished: BTreeSet, +} + +#[derive(Default)] +struct Tracking { + accepted: Mutex, + finished: Notify, +} + +struct Completion { + tracking: Arc, + id: u64, +} + +impl Drop for Completion { + fn drop(&mut self) { + self.tracking + .accepted + .lock() + .unwrap() + .unfinished + .remove(&self.id); + self.tracking.finished.notify_waiters(); + } +} + +struct Job { + operation: Box, + // Field drop order releases authority before completion becomes observable. + _lifetime: Option>, + lifetime_source: Option>, + _completion: Completion, + runtime: Option, + diagnostics: tracing::Dispatch, + span: tracing::Span, +} + +fn report_unknown(accepted_job_id: u64, phase: &'static str) { + tracing::error!( + code = "db_operation_outcome_unknown", + accepted_job_id, + phase, + error = %Error::DbOperationOutcomeUnknown, + "DB job panicked; outcome unknown, operation will not be retried" + ); +} + +/// Clones share the queue and actual-job budget. Workers retain only the +/// receiver, so dropping the last sender closes the queue; accepted jobs finish +/// before the dedicated threads exit. No thread is created per store open. +#[derive(Clone)] +pub struct DbExecutor { + sender: Option>, + tracking: Arc, + max_jobs: usize, +} + +impl DbExecutor { + pub fn new(limits: DbExecutorLimits) -> Result { + let max_jobs = limits + .workers + .get() + .checked_add(limits.queued.get()) + .ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?; + let (sender, receiver) = mpsc::sync_channel::(limits.queued.get()); + let receiver = Arc::new(Mutex::new(receiver)); + for index in 0..limits.workers.get() { + let receiver = receiver.clone(); + std::thread::Builder::new() + .name(format!("workspace-db-{index}")) + .spawn(move || { + loop { + let next = receiver.lock().unwrap().recv(); + let Ok(job) = next else { break }; + let Job { + operation, + _lifetime: lifetime, + lifetime_source, + _completion: completion, + runtime, + diagnostics, + span, + } = job; + // Keep diagnostics with their submitting subscriber as + // well as the runtime; no process-global override. + let _diagnostics = tracing::dispatcher::set_default(&diagnostics); + let _span = span.enter(); + // Preserve the submitting runtime for synchronous Host + // post-commit hooks, including reply/lease destructors. + // enter() supplies context only: never drive a future + // or await/nest an executor submission in a transaction. + let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter); + // A cancelled waiter's discarded result may itself have + // a destructor. Keep both the lease and actual-job slot + // until that cleanup finishes, even if it panics. + let execute = || { + if catch_unwind(AssertUnwindSafe(operation)).is_err() { + report_unknown(completion.id, "reply_cleanup"); + } + if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() { + report_unknown(completion.id, "lifetime_cleanup"); + } + }; + // Synchronous post-commit continuations can admit another + // bounded job. Transfer the original lifecycle source as + // well, so its authority survives the parent job/waiter. + // This does not inject context into detached Tokio tasks. + if let Some(source) = lifetime_source { + OPERATION_LIFETIME.sync_scope(source, execute); + } else { + execute(); + } + drop(completion); + } + }) + .map_err(|_| Error::DbExecutorUnavailable)?; + } + Ok(Self { + sender: Some(sender), + tracking: Arc::default(), + max_jobs, + }) + } + + pub fn shared() -> Self { + static SHARED: OnceLock = OnceLock::new(); + SHARED + .get_or_init(|| { + Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self { + sender: None, + tracking: Arc::default(), + max_jobs: 0, + }) + }) + .clone() + } + + /// Submit without waiting for queue capacity. The closure must contain the + /// entire synchronous operation/transaction, with no nested async runtime. + /// The submitting Tokio handle, when present, is entered for synchronous + /// post-commit hooks and worker cleanup; it is never used to block_on SQL. + pub async fn run( + &self, + operation: impl FnOnce() -> std::result::Result + Send + 'static, + ) -> std::result::Result + where + T: Send + 'static, + E: From + Send + 'static, + { + self.submit(operation)? + .await + .unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into())) + } + + /// Admit a fire-and-forget operation immediately, without spawning a Tokio + /// task or waiting for capacity. Err means the operation was not accepted; + /// Ok means the worker owns it through execution and cleanup, even after + /// the submitting task is cancelled. A panic emits a structured unknown- + /// outcome diagnostic, never an automatic retry. drain() tracks these jobs. + pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> { + self.submit(move || { + operation(); + Ok::<_, Error>(()) + }) + .map(drop) + } + + /// Shared synchronous admission: neither dropped receivers nor cancelled + /// callers remove accepted work from the queue or the actual-job budget. + fn submit( + &self, + operation: impl FnOnce() -> std::result::Result + Send + 'static, + ) -> std::result::Result>, E> + where + T: Send + 'static, + E: From + Send + 'static, + { + let sender = self + .sender + .as_ref() + .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; + let runtime = tokio::runtime::Handle::try_current().ok(); + let diagnostics = tracing::dispatcher::get_default(Clone::clone); + let span = tracing::Span::current(); + let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok(); + let lifetime = lifetime_source.as_ref().map(|source| source.retain()); + let (reply, result) = oneshot::channel(); + // Serialize admission and acceptance snapshots. Worker completion cannot + // race acceptance, and cancelled waiters never alter this actual-job cap. + { + let mut accepted = self.tracking.accepted.lock().unwrap(); + if accepted.unfinished.len() >= self.max_jobs { + return Err(Error::DbCapacityExceeded.into()); + } + let id = accepted + .last_job_id + .checked_add(1) + .ok_or_else(|| E::from(Error::DbExecutorUnavailable))?; + // Never reuse the identity of a rejected job: its destructor may run + // after another submission has acquired the admission lock. + accepted.last_job_id = id; + let job = Job { + operation: Box::new(move || { + let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| { + report_unknown(id, "operation"); + Err(Error::DbOperationOutcomeUnknown.into()) + }); + let _ = reply.send(outcome); + }), + _lifetime: lifetime, + lifetime_source, + runtime, + diagnostics, + span, + _completion: Completion { + tracking: self.tracking.clone(), + id, + }, + }; + match sender.try_send(job) { + Ok(()) => { + accepted.unfinished.insert(id); + } + Err(error) => { + // The rejected Job's completion also locks tracking; drop it only + // after releasing the admission lock. + drop(accepted); + return Err(match error { + mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded, + mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable, + } + .into()); + } + } + } + Ok(result) + } + + /// Wait for every job accepted before this barrier is first polled (not + /// later submissions). This bypasses queue capacity and survives cancelled + /// job waiters. It does not close the queue or stop new submissions. + pub async fn drain(&self) -> Result<()> { + if self.sender.is_none() { + return Err(Error::DbExecutorUnavailable); + } + let through = self.tracking.accepted.lock().unwrap().last_job_id; + loop { + let notified = self.tracking.finished.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self + .tracking + .accepted + .lock() + .unwrap() + .unfinished + .first() + .is_none_or(|id| *id > through) + { + return Ok(()); + } + notified.await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::{Connection, TransactionBehavior, ffi}; + use std::{ffi::c_void, time::Duration}; + + fn executor(queued: usize) -> DbExecutor { + DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(queued).unwrap(), + }) + .unwrap() + } + + fn database() -> (tempfile::TempDir, Connection, Connection) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite"); + let owner = Connection::open(&path).unwrap(); + owner + .execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)") + .unwrap(); + let worker = Connection::open(&path).unwrap(); + (dir, owner, worker) + } + + struct BusyGate { + waiting: Option>, + release: mpsc::Receiver<()>, + } + + // A fixture-owned callback proves SQLite actually reached its lock wait. + // Unlike busy_handler's fn pointer API, the SQLite API carries a per-call + // context, so tests need neither process-global state nor guessed sleeps. + struct BusyRegistration { + db: *mut ffi::sqlite3, + _gate: Box, + } + + impl BusyRegistration { + fn install(conn: &Connection, gate: BusyGate) -> Self { + unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 { + // The Box remains registered only while this connection and + // BusyRegistration are alive, on this same dedicated worker. + let gate = unsafe { &mut *context.cast::() }; + if let Some(waiting) = gate.waiting.take() { + let _ = waiting.send(()); + if gate.release.recv().is_ok() { + return 1; + } + } + 0 + } + let mut gate = Box::new(gate); + // SAFETY: conn owns the handle; registration is dropped before conn. + let db = unsafe { conn.handle() }; + let result = unsafe { + ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast()) + }; + assert_eq!(result, ffi::SQLITE_OK); + Self { db, _gate: gate } + } + } + + impl Drop for BusyRegistration { + fn drop(&mut self) { + // SAFETY: unregister before releasing the callback's boxed context. + unsafe { + ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut()); + } + } + } + + async fn observed(future: impl Future) -> T { + tokio::time::timeout(Duration::from_secs(5), future) + .await + .expect("DB job did not reach the expected observable state") + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() { + let (_dir, mut owner, mut worker) = database(); + let lock = owner + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let executor = executor(1); + let (waiting, blocked) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut write = Box::pin(executor.run(move || -> Result<()> { + let _busy = BusyRegistration::install( + &worker, + BusyGate { + waiting: Some(waiting), + release: permit, + }, + ); + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + Ok(()) + })); + assert!(futures::poll!(&mut write).is_pending()); + observed(blocked).await.unwrap(); + let control = tokio::spawn(async { 42 }); + assert_eq!(observed(control).await.unwrap(), 42); + // The timer itself is the contract under test, not synchronization. + observed(tokio::time::sleep(Duration::from_millis(1))).await; + assert!(futures::poll!(&mut write).is_pending()); + lock.rollback().unwrap(); + release.send(()).unwrap(); + observed(write).await.unwrap(); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + } + + struct Lease(Arc); + struct Retained(Arc); + impl DbOperationLifetime for Lease { + fn retain(&self) -> Box { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Box::new(Retained(self.0.clone())) + } + } + impl Drop for Retained { + fn drop(&mut self) { + self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() { + let (_dir, mut owner, mut worker) = database(); + let lock = owner + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let executor = executor(1); + let (waiting, blocked) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut running = Box::pin(executor.run(move || -> Result<()> { + let _busy = BusyRegistration::install( + &worker, + BusyGate { + waiting: Some(waiting), + release: permit, + }, + ); + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + Ok(()) + })); + assert!(futures::poll!(&mut running).is_pending()); + observed(blocked).await.unwrap(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap(); + let mut queued = Box::pin(with_operation_lifetime( + lease, + executor.run(move || -> Result<()> { + let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + // FIFO is observable in durable SQL, not inferred from waiter order. + let first: i64 = + tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| { + row.get(0) + })?; + assert_eq!(first, 1); + tx.execute("INSERT INTO writes VALUES (2)", [])?; + tx.commit()?; + Ok(()) + }), + )); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!(matches!( + executor.run(|| Ok::<_, Error>(())).await, + Err(Error::DbCapacityExceeded) + )); + let mut drained = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + lock.rollback().unwrap(); + release.send(()).unwrap(); + observed(running).await.unwrap(); + observed(drained).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() { + let (dir, owner, mut parent_conn) = database(); + let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap(); + let parent = executor(1); + let child = executor(1); + let child_owner = child.clone(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let source: Arc = Arc::new(Lease(count.clone())); + let (entered, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let (child_entered, child_running) = oneshot::channel(); + let (finish_child, child_permit) = mpsc::channel(); + let mut operation = Box::pin(with_operation_lifetime( + source, + parent.run(move || -> Result<()> { + let _ = entered.send(()); + let _ = permit.recv(); + let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + child_owner.dispatch(move || { + let tx = child_conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let committed: i64 = tx + .query_row("SELECT count(*) FROM writes", [], |row| row.get(0)) + .unwrap(); + assert_eq!(committed, 1, "child must see its parent's committed SQL"); + let _ = child_entered.send(()); + let _ = child_permit.recv(); + tx.execute("INSERT INTO writes VALUES (2)", []).unwrap(); + tx.commit().unwrap(); + })?; + Ok(()) + }), + )); + assert!(futures::poll!(&mut operation).is_pending()); + observed(running).await.unwrap(); + drop(operation); + release.send(()).unwrap(); + observed(child_running).await.unwrap(); + observed(parent.drain()).await.unwrap(); + assert_eq!( + count.load(std::sync::atomic::Ordering::SeqCst), + 1, + "accepted continuation lost authority when its parent finished" + ); + let mut drained = Box::pin(child.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + finish_child.send(()).unwrap(); + observed(drained).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); + } + + #[tokio::test] + async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let error = executor + .run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.execute("INSERT INTO writes VALUES (NULL)", [])?; + tx.commit()?; + Ok(()) + }) + .await + .unwrap_err(); + assert!(matches!(error, Error::Sqlite(_))); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test] + async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let error = executor + .run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + panic!("lost outcome after commit"); + }) + .await + .unwrap_err(); + assert!(matches!(error, Error::DbOperationOutcomeUnknown)); + executor.drain().await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42); + } + + #[tokio::test] + async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let error = with_operation_lifetime( + lease, + executor.run(move || -> Result<()> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + panic!("lost outcome before commit"); + }), + ) + .await + .unwrap_err(); + assert!(matches!(error, Error::DbOperationOutcomeUnknown)); + executor.drain().await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 0 + ); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + struct DiscardedReplyHook { + expected_runtime: tokio::runtime::Id, + lease_count: Arc, + dropped: Option>, + } + + impl Drop for DiscardedReplyHook { + fn drop(&mut self) { + let runtime = + tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context"); + let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); + let same_runtime = runtime.id() == self.expected_runtime; + let dropped = self.dropped.take().unwrap(); + tokio::spawn(async move { + let _ = dropped.send((same_runtime, retained)); + }); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop() + { + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut blocker = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut blocker).is_pending()); + observed(running).await.unwrap(); + let expected_runtime = tokio::runtime::Handle::current().id(); + let (hook, published) = oneshot::channel(); + let (finish, finishing) = mpsc::channel(); + let (dropped, reply_dropped) = oneshot::channel(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let operation_count = count.clone(); + let mut queued = Box::pin(with_operation_lifetime( + lease, + executor.run(move || -> Result<_> { + let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + // Model synchronous Host hooks: publish only after commit, checking + // try_current and spawning without driving any future on this worker. + let runtime = tokio::runtime::Handle::try_current() + .expect("post-commit hook lost runtime context"); + assert_eq!(runtime.id(), expected_runtime); + tokio::spawn(async move { + let _ = hook.send(()); + }); + let _ = finishing.recv(); + Ok(DiscardedReplyHook { + expected_runtime, + lease_count: operation_count, + dropped: Some(dropped), + }) + }), + )); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + release.send(()).unwrap(); + observed(blocker).await.unwrap(); + observed(published).await.unwrap(); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + finish.send(()).unwrap(); + assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1)); + observed(drain).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() { + let executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(2).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let mut releases = Vec::new(); + // Observe each worker executing before submitting the next operation, + // so queue occupancy and overload are deterministic, not timing guesses. + for _ in 0..2 { + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut operation = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut operation).is_pending()); + observed(running).await.unwrap(); + drop(operation); + releases.push(release); + } + let (finished, mut queued_finished) = oneshot::channel(); + let mut queued = Box::pin(executor.run(move || -> Result<()> { + let _ = finished.send(()); + Ok(()) + })); + assert!(futures::poll!(&mut queued).is_pending()); + drop(queued); + let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + for _ in 0..8 { + let rejected_runs = rejected_runs.clone(); + assert!(matches!( + executor + .run(move || { + rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok::<_, Error>(()) + }) + .await, + Err(Error::DbCapacityExceeded) + )); + } + assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(matches!( + queued_finished.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + for release in releases { + release.send(()).unwrap(); + } + observed(drain).await.unwrap(); + observed(queued_finished).await.unwrap(); + executor.run(|| Ok::<_, Error>(())).await.unwrap(); + } + + #[derive(Default)] + struct UnknownDiagnostic { + code: Option, + phase: Option, + accepted_job_id: Option, + } + + impl tracing::field::Visit for UnknownDiagnostic { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + match field.name() { + "code" => self.code = Some(value.to_owned()), + "phase" => self.phase = Some(value.to_owned()), + _ => {} + } + } + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + if field.name() == "accepted_job_id" { + self.accepted_job_id = Some(value); + } + } + fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} + } + + #[derive(Clone, Default)] + struct DiagnosticCapture(Arc>>); + + impl tracing_subscriber::Layer for DiagnosticCapture { + fn on_event( + &self, + event: &tracing::Event<'_>, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut diagnostic = UnknownDiagnostic::default(); + event.record(&mut diagnostic); + self.0.lock().unwrap().push(diagnostic); + } + } + + struct DispatchCleanup { + expected_runtime: tokio::runtime::Id, + lease_count: Arc, + started: Option>, + release: mpsc::Receiver<()>, + } + + impl Drop for DispatchCleanup { + fn drop(&mut self) { + // No panicking assertions during unwind: report observations to the + // test, and gate cleanup so drain/lifetime behavior is observable. + let same_runtime = tokio::runtime::Handle::try_current() + .is_ok_and(|runtime| runtime.id() == self.expected_runtime); + let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst); + if let Some(started) = self.started.take() { + let _ = started.send((same_runtime, retained)); + } + let _ = self.release.recv(); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry() + { + use tracing_subscriber::layer::SubscriberExt; + let (_dir, owner, mut worker) = database(); + let executor = executor(1); + let (started, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let mut blocker = Box::pin(executor.run(move || -> Result<()> { + let _ = started.send(()); + let _ = permit.recv(); + Ok(()) + })); + assert!(futures::poll!(&mut blocker).is_pending()); + observed(running).await.unwrap(); + drop(blocker); // Cancellation does not free its running slot. + let expected_runtime = tokio::runtime::Handle::current().id(); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let lease: Arc = Arc::new(Lease(count.clone())); + let operation_count = count.clone(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let operation_attempts = attempts.clone(); + let (cleanup_started, cleaning_up) = oneshot::channel(); + let (finish, finishing) = mpsc::channel(); + // Model multiple tracing contexts without setting a global subscriber. + // This also prevents tracing's single-subscriber callsite optimization + // from caching an unsubscribed parallel test's Interest::never here. + let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry()); + let captured = DiagnosticCapture::default(); + let diagnostics = + tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone())); + with_operation_lifetime(lease, async { + tracing::dispatcher::with_default(&diagnostics, || { + executor.dispatch(move || { + let _cleanup = DispatchCleanup { + expected_runtime, + lease_count: operation_count, + started: Some(cleanup_started), + release: finishing, + }; + operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let tx = worker + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + tx.execute("INSERT INTO writes VALUES (1)", []).unwrap(); + tx.commit().unwrap(); + panic!("dispatched post-commit outcome lost"); + }) + }) + }) + .await + .unwrap(); // Dispatch is admitted synchronously; no waiter survives. + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let rejected_operation = rejected_runs.clone(); + assert!(matches!( + executor.dispatch(move || { + rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }), + Err(Error::DbCapacityExceeded) + )); + release.send(()).unwrap(); + assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1)); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + let mut drain = Box::pin(executor.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + finish.send(()).unwrap(); + observed(drain).await.unwrap(); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0); + { + let events = captured.0.lock().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!( + events[0].code.as_deref(), + Some("db_operation_outcome_unknown") + ); + assert_eq!(events[0].phase.as_deref(), Some("operation")); + assert!(events[0].accepted_job_id.is_some()); + } + let (completed, completion) = oneshot::channel(); + executor + .dispatch(move || { + let _ = completed.send(()); + }) + .unwrap(); + observed(executor.drain()).await.unwrap(); + observed(completion).await.unwrap(); + } +} diff --git a/crates/workspace-server/src/frozen_content_state_tests.rs b/crates/workspace-server/src/frozen_content_state_tests.rs index 138b83683..954d42ccf 100644 --- a/crates/workspace-server/src/frozen_content_state_tests.rs +++ b/crates/workspace-server/src/frozen_content_state_tests.rs @@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store @@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { migrate_content_state_v85_to_v86(&conn).unwrap(); drop(conn); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), }; let job = store diff --git a/crates/workspace-server/src/frozen_repository_key_tests.rs b/crates/workspace-server/src/frozen_repository_key_tests.rs index adeef98fe..798f1575d 100644 --- a/crates/workspace-server/src/frozen_repository_key_tests.rs +++ b/crates/workspace-server/src/frozen_repository_key_tests.rs @@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }; let migrated = store @@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let archived = store @@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = @@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() { // The service consumes current receipt columns; unrelated schemas are not this fixture. rename_saved_secret_receipt_counters(&conn).unwrap(); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }); let service = diff --git a/crates/workspace-server/src/frozen_runtime_binding_tests.rs b/crates/workspace-server/src/frozen_runtime_binding_tests.rs index fcf7dcc97..a23c29b04 100644 --- a/crates/workspace-server/src/frozen_runtime_binding_tests.rs +++ b/crates/workspace-server/src/frozen_runtime_binding_tests.rs @@ -533,6 +533,7 @@ mod existing_workspace_provisioning { // This fixture contains only the schema-87 cutover authorities, not the // unrelated full Server schema. Exercise the real service/store methods. Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(conn)), }) } @@ -644,6 +645,7 @@ mod existing_workspace_provisioning { drop(service); drop(store); let store = Arc::new(SqliteWorkspaceStore { + db_executor: crate::db_executor::DbExecutor::shared(), conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())), }); let service = WorkspaceSigningIdentityService::new( diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index cb8dfa169..73733162b 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -23,7 +23,10 @@ use std::{ }; use workdir::{ Workdir, WorkdirError, WorkdirSessionHandle, - http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization}, + http::{ + OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization, + WorkdirHttpAuthorizationFuture, + }, }; use worker_runtime::RuntimeWorkspaceScope; use worker_runtime::catalog::{ @@ -3473,6 +3476,38 @@ impl WorkspaceRuntimeAuthorization { } } + /// Offload the complete live-binding check and request signing operation. + /// Claims are minted inside the bounded job, not cached before admission. + pub(crate) async fn issue_async( + &self, + method: String, + path_and_query: String, + operation: String, + worker_id: Option, + body: Vec, + ) -> Result { + let executor = self.store.db_executor(); + let authorization = self.clone(); + executor + .run(move || { + Ok::<_, Error>(authorization.issue( + &method, + &path_and_query, + &operation, + worker_id.as_deref(), + &body, + )) + }) + .await + .map_err(|error| { + diagnostic( + "workspace_runtime_authorization_unavailable", + HostDiagnosticSeverity::Error, + error.to_string(), + ) + })? + } + pub(crate) fn issue( &self, method: &str, @@ -3645,25 +3680,24 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization { impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization { fn bearer_token( &self, - method: &str, - path_and_query: &str, - body: &[u8], - ) -> Result { - if let Some(authorization) = &self.workspace_authorization { - return authorization - .issue( - method, - path_and_query, - workspace_runtime_operation(method, path_and_query), - None, - body, + method: String, + path_and_query: String, + body: Vec, + ) -> WorkdirHttpAuthorizationFuture { + let authorization = self.clone(); + Box::pin(async move { + if let Some(authorizer) = &authorization.workspace_authorization { + let operation = workspace_runtime_operation(&method, &path_and_query).to_string(); + return authorizer + .issue_async(method, path_and_query, operation, None, body) + .await + .map_err(|error| WorkdirError::Unavailable(error.message)); + } + authorization.fallback_bearer_token.ok_or_else(|| { + WorkdirError::Unavailable( + "remote Runtime does not have bearer authorization configured".to_string(), ) - .map_err(|error| WorkdirError::Unavailable(error.message)); - } - self.fallback_bearer_token.clone().ok_or_else(|| { - WorkdirError::Unavailable( - "remote Runtime does not have bearer authorization configured".to_string(), - ) + }) }) } } @@ -6344,6 +6378,79 @@ mod tests { use std::sync::{Arc, Mutex}; use std::thread; + #[tokio::test] + async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() { + use crate::db_executor::{DbExecutor, DbExecutorLimits}; + use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding}; + use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore; + + let executor = DbExecutor::new(DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::in_memory() + .unwrap() + .with_db_executor(executor.clone()), + ); + let service = WorkspaceSigningIdentityService::new( + store.clone(), + Arc::new(InMemoryWorkspaceSigningMaterialStore::default()), + ); + let authorization = RemoteWorkdirAuthorization { + workspace_authorization: Some(WorkspaceRuntimeAuthorization::new( + store, + service, + "https://backend.test", + WorkspaceRuntimeBinding { + workspace_id: "workspace".into(), + runtime_id: "runtime".into(), + display_name: "Runtime".into(), + base_url: "https://runtime.test".into(), + public_key_fingerprint: "unused".into(), + public_key: "unused".into(), + binding_id: "binding".into(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: "1".into(), + updated_at: "1".into(), + revoked_at: None, + }, + )), + fallback_bearer_token: Some("must-not-be-used".into()), + }; + + // Hold the running slot and fill the only queue slot. The callback must + // reject admission, not run inline or silently switch authentication. + let (release, wait) = std::sync::mpsc::channel::<()>(); + let (started, running) = tokio::sync::oneshot::channel(); + executor + .dispatch(move || { + let _ = started.send(()); + let _ = wait.recv_timeout(Duration::from_secs(5)); + }) + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), running) + .await + .unwrap() + .unwrap(); + executor.dispatch(|| {}).unwrap(); + let error = authorization + .bearer_token( + "POST".into(), + "/v1/workdir-sessions/s/operations".into(), + b"{}".to_vec(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string() + )); + drop(release); + executor.drain().await.unwrap(); + } + #[test] fn ticket_worker_spawn_intent_uses_worker_wire_role() { let intent = WorkerSpawnIntent::TicketRole { diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index fefe24d92..eb113b079 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -10,6 +10,7 @@ pub mod backend_job; pub mod companion; pub mod config; pub mod config_source; +pub mod db_executor; pub mod feature_storage; pub mod hosts; pub mod memory_backend; @@ -159,6 +160,12 @@ pub enum Error { WorkspaceSigningIdentity { code: String, message: String }, #[error("Workspace Drive error: {0}")] Drive(#[from] workspace_drive::Error), + #[error("DB execution capacity exceeded")] + DbCapacityExceeded, + #[error("DB executor unavailable")] + DbExecutorUnavailable, + #[error("DB operation outcome unknown; do not automatically retry")] + DbOperationOutcomeUnknown, #[error("store error: {0}")] Store(String), } diff --git a/crates/workspace-server/src/runtime_subscription.rs b/crates/workspace-server/src/runtime_subscription.rs index 3a7fdd0bd..998786c38 100644 --- a/crates/workspace-server/src/runtime_subscription.rs +++ b/crates/workspace-server/src/runtime_subscription.rs @@ -965,7 +965,7 @@ async fn connect_runtime( let mut request = endpoint .into_client_request() .map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?; - if let Some(token) = runtime_token(config, workspace_id)? { + if let Some(token) = runtime_token(config, workspace_id).await? { request.headers_mut().insert( "authorization", format!("Bearer {token}") @@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String { format!("{base}/v1/protocol/ws") } } -fn runtime_token( +async fn runtime_token( config: &RemoteRuntimeConfig, _workspace_id: &str, ) -> Result, String> { if let Some(authorization) = config.workspace_authorization.as_ref() { return authorization - .issue("GET", "/v1/protocol/ws", "workers:list", None, &[]) + .issue_async( + "GET".to_string(), + "/v1/protocol/ws".to_string(), + "workers:list".to_string(), + None, + Vec::new(), + ) + .await .map(Some) .map_err(|error| error.message); } diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 6740bc570..f6535f9c8 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -885,7 +885,7 @@ impl ExternalProviderWorkdirSession { } } - fn audit_operation( + async fn audit_operation( &self, operation_id: &ExternalWorkdirOperationId, operation_kind: &str, @@ -894,25 +894,34 @@ impl ExternalProviderWorkdirSession { let Some(audit) = &self.audit else { return Ok(()); }; + let audit = audit.clone(); + let operation_id = operation_id.clone(); + let operation_kind = operation_kind.to_owned(); + let outcome = outcome.to_owned(); + let connection = self.connection.clone(); audit .store - .record_external_workdir_audit( - &audit.workspace_id, - "worker", - &audit.actor_key, - "external_workdir_operation", - &self.connection.grant_id, - Some(operation_id.as_str()), - outcome, - &format!( - "workdir_id={} attachment_alias={} operation_kind={} generation={}", - self.connection.workdir_id, - audit.attachment_alias, - operation_kind, - self.connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) + .db_executor() + .run(move || { + audit.store.record_external_workdir_audit( + &audit.workspace_id, + "worker", + &audit.actor_key, + "external_workdir_operation", + &connection.grant_id, + Some(operation_id.as_str()), + &outcome, + &format!( + "workdir_id={} attachment_alias={} operation_kind={} generation={}", + connection.workdir_id, + audit.attachment_alias, + operation_kind, + connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await .map_err(|error| { workdir::WorkdirError::Unavailable(format!( "External Workdir operation audit failed: {error}" @@ -960,7 +969,8 @@ impl ExternalProviderWorkdirSession { } let operation_id = ExternalWorkdirOperationId::new(format!("op-{}", Uuid::now_v7())) .map_err(workdir::WorkdirError::InvalidArgument)?; - self.audit_operation(&operation_id, operation_kind, "requested")?; + self.audit_operation(&operation_id, operation_kind, "requested") + .await?; let (response, receive) = tokio::sync::oneshot::channel(); let mut cancel_guard = ExternalOperationCancelGuard { sender: self.connection.sender.clone(), @@ -977,7 +987,8 @@ impl ExternalProviderWorkdirSession { .await .is_err() { - self.audit_operation(&operation_id, operation_kind, "provider_unavailable")?; + self.audit_operation(&operation_id, operation_kind, "provider_unavailable") + .await?; return Err(workdir::WorkdirError::Unavailable( "External Workdir provider disconnected".to_string(), )); @@ -1007,7 +1018,8 @@ impl ExternalProviderWorkdirSession { } else { "failed" }, - )?; + ) + .await?; result } @@ -1851,6 +1863,27 @@ impl WorkerRemovalService { target_runtime_id: &str, target_worker_id: &str, reason: &str, + ) -> std::result::Result { + let service = self.clone(); + let runtime_id = target_runtime_id.to_owned(); + let worker_id = target_worker_id.to_owned(); + let reason = reason.to_owned(); + run_host_workflow(async move { + service + .execute_async_on_host(source, &runtime_id, &worker_id, &reason) + .await + .map_err(Error::Store) + }) + .await + .map_err(|error| error.to_string()) + } + + async fn execute_async_on_host( + &self, + source: crate::worker_source::VerifiedWorkerMutationSource, + target_runtime_id: &str, + target_worker_id: &str, + reason: &str, ) -> std::result::Result { let reason = reason.trim(); if reason.is_empty() || reason.len() > 512 { @@ -1922,11 +1955,36 @@ impl WorkerRemovalService { )); } - self.execute_target_removal(&runtime, &target, reason, None) + self.execute_target_removal_on_host(&runtime, &target, reason, None) .await } async fn execute_target_removal( + &self, + _runtime: &RuntimeRegistry, + target: &RuntimeWorkerRef, + reason: &str, + parent_workspace_deletion_operation_id: Option<&str>, + ) -> std::result::Result { + let runtime = self + .runtime + .upgrade() + .ok_or_else(|| "Workspace Runtime registry is unavailable".to_string())?; + let service = self.clone(); + let target = target.clone(); + let reason = reason.to_owned(); + let parent = parent_workspace_deletion_operation_id.map(str::to_owned); + run_host_workflow(async move { + service + .execute_target_removal_on_host(&runtime, &target, &reason, parent.as_deref()) + .await + .map_err(Error::Store) + }) + .await + .map_err(|error| error.to_string()) + } + + async fn execute_target_removal_on_host( &self, runtime: &RuntimeRegistry, target: &RuntimeWorkerRef, @@ -2220,6 +2278,16 @@ impl WorkspaceServerApi { } } + // Own the complete synchronous store stage; callers retain their async + // lifecycle guards across it and never split a transaction around await. + async fn execute_store_stage( + &self, + operation: impl FnOnce(Arc) -> Result + Send + 'static, + ) -> Result { + let store = self.store.clone(); + self.store.db_executor().run(move || operation(store)).await + } + async fn admission_for_workspace(&self, workspace_id: &str) -> Arc { workspace_admission_gate(&self.admission_gates, workspace_id).await } @@ -2232,7 +2300,13 @@ impl WorkspaceServerApi { let Some(workspace) = self.store.get_workspace(workspace_id).await? else { return Ok(None); }; - let repositories = self.store.list_repositories(workspace_id)?; + let store = self.store.clone(); + let id = workspace_id.to_owned(); + let repositories = self + .store + .db_executor() + .run(move || store.list_repositories(&id)) + .await?; let config = self .template .for_catalog_workspace(&workspace, repositories)?; @@ -2249,34 +2323,46 @@ impl WorkspaceServerApi { drop(running); let api = self.clone(); tokio::spawn(async move { - if api.execute_workspace_deletion(&operation_id).await.is_err() { - let operation = api - .store - .workspace_deletion_operation_for_recovery(&operation_id) - .ok() - .flatten(); - let child_operation_ids = operation - .as_ref() - .map(|operation| operation.child_operation_ids.as_slice()) - .unwrap_or_default(); - let blockers = operation - .as_ref() - .map(|operation| operation.blockers.as_slice()) - .unwrap_or_default(); - let _ = api.store.update_workspace_deletion_operation( - &operation_id, - WorkspaceDeletionState::Failed, - child_operation_ids, - blockers, - Some("workspace_deletion_execution_failed"), - ); + if let Err(error) = api.execute_workspace_deletion(&operation_id).await { + tracing::warn!(%error, %operation_id, "Workspace deletion execution did not complete"); + if !matches!(error, Error::DbOperationOutcomeUnknown) { + let id = operation_id.clone(); + let _ = api + .execute_store_stage(move |store| { + let operation = store + .workspace_deletion_operation_for_recovery(&id) + .ok() + .flatten(); + let child_operation_ids = operation + .as_ref() + .map(|operation| operation.child_operation_ids.as_slice()) + .unwrap_or_default(); + let blockers = operation + .as_ref() + .map(|operation| operation.blockers.as_slice()) + .unwrap_or_default(); + store + .update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Failed, + child_operation_ids, + blockers, + Some("workspace_deletion_execution_failed"), + ) + .map(drop) + }) + .await; + } } api.running_deletions.lock().await.remove(&operation_id); }); } async fn recover_workspace_deletions(&self) -> Result<()> { - for operation_id in self.store.resumable_workspace_deletion_operation_ids()? { + let operations = self + .execute_store_stage(|store| store.resumable_workspace_deletion_operation_ids()) + .await?; + for operation_id in operations { self.schedule_workspace_deletion(operation_id).await; } Ok(()) @@ -2287,16 +2373,20 @@ impl WorkspaceServerApi { actor_account_id: &str, workspace_id: &str, ) -> Result { + let actor = actor_account_id.to_owned(); + let id = workspace_id.to_owned(); let mut preflight = self - .store - .workspace_deletion_preflight(actor_account_id, workspace_id)?; + .execute_store_stage(move |store| store.workspace_deletion_preflight(&actor, &id)) + .await?; let Some(api) = self.api_for_workspace(workspace_id).await? else { return Err(Error::InvalidInput("Workspace does not exist".to_string())); }; - for registry_worker in self - .store - .list_worker_registry(workspace_id, i64::MAX as usize)? - { + let id = workspace_id.to_owned(); + let workers = self + .execute_store_stage(move |store| store.list_worker_registry(&id, i64::MAX as usize)) + .await?; + host_io_executor()?.run(move || -> Result<_> { + for registry_worker in workers { let worker_key = registry_worker.display_name; match api.runtime.worker(®istry_worker.worker) { Ok(worker) if worker.state == "stopped" => {} @@ -2319,23 +2409,30 @@ impl WorkspaceServerApi { crate::workspace_deletion::bound_workspace_deletion_blockers(&mut preflight.blockers); preflight.can_delete = preflight.blockers.is_empty(); Ok(preflight) + }).await } async fn execute_workspace_deletion( &self, operation_id: &str, ) -> Result { - let prior = self - .store - .workspace_deletion_operation_for_recovery(operation_id)? - .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; - let operation = self.store.update_workspace_deletion_operation( - operation_id, - WorkspaceDeletionState::Running, - &prior.child_operation_ids, - &[], - None, - )?; + let id = operation_id.to_owned(); + let operation = self + .execute_store_stage(move |store| { + let prior = store + .workspace_deletion_operation_for_recovery(&id)? + .ok_or_else(|| { + Error::InvalidInput("Workspace deletion operation".to_string()) + })?; + store.update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Running, + &prior.child_operation_ids, + &[], + None, + ) + }) + .await?; let api = self .api_for_workspace(&operation.workspace_id) .await? @@ -2344,10 +2441,13 @@ impl WorkspaceServerApi { api.fence_drive().await?; let mut child_operation_ids = operation.child_operation_ids.clone(); let mut blockers = Vec::new(); - for worker in self - .store - .list_worker_registry(&operation.workspace_id, i64::MAX as usize)? - { + let workspace_id = operation.workspace_id.clone(); + let workers = self + .execute_store_stage(move |store| { + store.list_worker_registry(&workspace_id, i64::MAX as usize) + }) + .await?; + for worker in workers { let worker_key = worker.display_name.clone(); let target = worker.worker; let response = WorkerRemovalService::new(&api) @@ -2359,13 +2459,17 @@ impl WorkspaceServerApi { ) .await .map_err(Error::Store)?; + let id = operation_id.to_owned(); child_operation_ids = self - .store - .workspace_deletion_operation_for_recovery(operation_id)? - .ok_or_else(|| { - Error::Store("Workspace deletion operation disappeared".to_string()) - })? - .child_operation_ids; + .execute_store_stage(move |store| { + store + .workspace_deletion_operation_for_recovery(&id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + }) + .map(|operation| operation.child_operation_ids) + }) + .await?; if response.status != 200 { blockers.push(WorkspaceDeletionBlocker { kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, @@ -2377,22 +2481,34 @@ impl WorkspaceServerApi { } if blockers.is_empty() { - for workdir in self - .store - .list_workdir_registry(&operation.workspace_id, i64::MAX as usize)? - { - match execute_workdir_removal_for_workspace_deletion( - &api, - &workdir.workdir_id, - operation_id, - ) { + let workspace_id = operation.workspace_id.clone(); + let workdirs = self + .execute_store_stage(move |store| { + store.list_workdir_registry(&workspace_id, i64::MAX as usize) + }) + .await?; + for workdir in workdirs { + let removal_api = api.clone(); + let workdir_id = workdir.workdir_id.clone(); + let parent_id = operation_id.to_owned(); + match host_io_executor()? + .run(move || { + execute_workdir_removal_for_workspace_deletion( + &removal_api, + &workdir_id, + &parent_id, + ) + }) + .await + { Ok(child) => { + let id = operation_id.to_owned(); + let child_id = child.operation_id.clone(); child_operation_ids = self - .store - .append_workspace_deletion_child_operation( - operation_id, - &child.operation_id, - )? + .execute_store_stage(move |store| { + store.append_workspace_deletion_child_operation(&id, &child_id) + }) + .await? .child_operation_ids; if child.state != WorkdirRemovalOperationState::Completed || child.disposition != Some(WorkdirRemovalDisposition::Removed) @@ -2428,29 +2544,40 @@ impl WorkspaceServerApi { } if !blockers.is_empty() { - return self.store.update_workspace_deletion_operation( - operation_id, - WorkspaceDeletionState::Blocked, - &child_operation_ids, - &blockers, - None, - ); + let id = operation_id.to_owned(); + return self + .execute_store_stage(move |store| { + store.update_workspace_deletion_operation( + &id, + WorkspaceDeletionState::Blocked, + &child_operation_ids, + &blockers, + None, + ) + }) + .await; } api.purge_drive().await?; let feature_storage = api.feature_storage.clone(); - tokio::task::spawn_blocking(move || feature_storage.delete()) - .await - .map_err(|error| { - Error::Store(format!("Feature storage deletion task failed: {error}")) - })? - .map_err(|error| { - Error::Store(format!( - "failed to delete Workspace Feature storage: {error}" - )) - })?; - WorkspaceSigningIdentityService::new(self.store.clone(), self.signing_materials.clone()) - .delete_material(&operation.workspace_id)?; - let completed = self.store.finalize_workspace_deletion(operation_id)?; + api.store + .db_executor() + .run(move || { + feature_storage.delete().map_err(|error| { + Error::Store(format!( + "failed to delete Workspace Feature storage: {error}" + )) + }) + }) + .await?; + let materials = self.signing_materials.clone(); + let id = operation_id.to_owned(); + let completed = self + .execute_store_stage(move |store| { + WorkspaceSigningIdentityService::new(store.clone(), materials) + .delete_material(&operation.workspace_id)?; + store.finalize_workspace_deletion(&id) + }) + .await?; self.routers.lock().await.remove(&completed.workspace_id); if let Some(handle) = self .hook_handles @@ -2497,15 +2624,28 @@ impl WorkspaceServerApi { async fn shutdown_feature_storage(&self) -> Result<()> { let apis = self.apis.lock().await.values().cloned().collect::>(); for api in apis { - api.feature_storage.shutdown().map_err(|error| { - Error::Store(format!("failed to shut down Feature storage: {error}")) - })?; + api.worker_projection.shutdown_async().await; + let storage = api.feature_storage.clone(); + api.store + .db_executor() + .run(move || { + storage.shutdown().map_err(|error| { + Error::Store(format!("failed to shut down Feature storage: {error}")) + }) + }) + .await?; } Ok(()) } async fn preload(&self) -> Result<()> { - for workspace in self.store.list_workspaces()? { + let store = self.store.clone(); + let workspaces = self + .store + .db_executor() + .run(move || store.list_workspaces()) + .await?; + for workspace in workspaces { let _ = self .router_for_workspace(&workspace.workspace_id) .await? @@ -2575,7 +2715,7 @@ async fn resolve_server_actor( headers: &HeaderMap, ) -> std::result::Result, Error> { let cookie_name = auth_public_config(api.template.as_ref()).cookie_name; - resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await + resolve_request_actor(api.store.clone(), headers, &cookie_name).await } fn signed_request_target(uri: &Uri) -> &str { @@ -2614,6 +2754,11 @@ fn runtime_request_proof_rejection( workspace_id: &str, error: &crate::worker_source::RuntimeRequestProofError, ) -> Response { + let status = if error.is_availability_error() { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::UNAUTHORIZED + }; // Never format an error or raw claims. IDs are fixed-length fingerprints; // verification labels describe proof-check progress, not authorization. let path = path.split('?').next().unwrap_or(path); @@ -2625,7 +2770,7 @@ fn runtime_request_proof_rejection( target: "yoi::auth", event = "runtime_request_proof_rejected", method, path = log_path, workspace_id, request_id, - status = 401, + status = status.as_u16(), reason = error.reason, proof_verification = error.verification, claimed_runtime_id_hash = error.runtime_id_hash.as_deref(), @@ -2640,8 +2785,12 @@ fn runtime_request_proof_rejection( ); repository_api_rejection( path, - StatusCode::UNAUTHORIZED, - "invalid runtime request proof", + status, + if error.is_availability_error() { + "Runtime authentication authority unavailable" + } else { + "invalid runtime request proof" + }, ) } @@ -2716,7 +2865,7 @@ async fn authorize_scoped_workspace_request( worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION }; let source = crate::worker_source::verify_runtime_request_source_proof_with_clock( - api.store.as_ref(), + api.store.clone(), api.template.as_ref(), &proof, workspace_id, @@ -2881,41 +3030,38 @@ async fn authorize_workspace_api_request( if !matches!( *request.method(), Method::GET | Method::HEAD | Method::OPTIONS - ) && !api - .store - .get_workspace(&workspace_id) - .await - .ok() - .flatten() - .is_some_and(|workspace| workspace.state == "active") - { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); + ) { + let workspace = match api.store.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(error) => return server_error_response(error), + }; + if !workspace.is_some_and(|workspace| workspace.state == "active") { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); + } } return next.run(request).await; } let AuthConfig::Passkey { cookie_name, .. } = &api.config.auth; - let actor = match crate::auth::resolve_request_actor( - api.store.as_ref(), - request.headers(), - cookie_name, - ) - .await - { - Ok(Some(actor)) => actor, - Ok(None) if public_server_api => return next.run(request).await, - Ok(None) | Err(_) => { - return repository_api_rejection( - &request_path, - StatusCode::UNAUTHORIZED, - "authentication required", - ); - } - }; + let actor = + match crate::auth::resolve_request_actor(api.store.clone(), request.headers(), cookie_name) + .await + { + Ok(Some(actor)) => actor, + Ok(None) if public_server_api => return next.run(request).await, + Err(error) => return server_error_response(error), + Ok(None) => { + return repository_api_rejection( + &request_path, + StatusCode::UNAUTHORIZED, + "authentication required", + ); + } + }; let cookie_authenticated = matches!(actor.auth_method, ActorAuthMethod::BrowserSession); let mutating = !matches!( *request.method(), @@ -2938,20 +3084,18 @@ async fn authorize_workspace_api_request( } } request.extensions_mut().insert(actor); - if mutating - && !api - .store - .get_workspace(&workspace_id) - .await - .ok() - .flatten() - .is_some_and(|workspace| workspace.state == "active") - { - return repository_api_rejection( - &request_path, - StatusCode::CONFLICT, - "Workspace is deleting and no longer accepts mutations", - ); + if mutating { + let workspace = match api.store.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(error) => return server_error_response(error), + }; + if !workspace.is_some_and(|workspace| workspace.state == "active") { + return repository_api_rejection( + &request_path, + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ); + } } next.run(request).await } @@ -3002,7 +3146,7 @@ fn workspace_request_is_callback(method: &Method, path: &str, workspace_id: &str async fn dispatch_workspace_request( State(api): State, - mut request: Request, + request: Request, ) -> Response { let path = request.uri().path().to_owned(); let workspace_id = scoped_workspace_id(&path); @@ -3028,6 +3172,23 @@ async fn dispatch_workspace_request( } else { None }; + if let Some(lease) = _request_lease { + crate::db_executor::with_operation_lifetime( + Arc::new(lease), + dispatch_admitted_workspace_request(api, request), + ) + .await + } else { + dispatch_admitted_workspace_request(api, request).await + } +} + +async fn dispatch_admitted_workspace_request( + api: WorkspaceServerApi, + mut request: Request, +) -> Response { + let path = request.uri().path().to_owned(); + let workspace_id = scoped_workspace_id(&path); if let Some(workspace_id) = workspace_id && (path.starts_with("/api/w/") || path.starts_with("/api/runtime/v1/workspaces/") @@ -3044,7 +3205,13 @@ async fn dispatch_workspace_request( Err(error) => return server_error_response(error), } } else { - let workspaces = match api.store.list_workspaces() { + let store = api.store.clone(); + let workspaces = match api + .store + .db_executor() + .run(move || store.list_workspaces()) + .await + { Ok(workspaces) => workspaces, Err(error) => return server_error_response(error), }; @@ -3474,8 +3641,18 @@ impl WorkspaceApi { config.workspace_id ))); } - import_configured_repositories(store.as_ref(), &config)?; - config.repositories = load_configured_repositories_from_store(store.as_ref(), &config)?; + let repository_store = store.clone(); + let repository_config = config.clone(); + config.repositories = store + .db_executor() + .run(move || { + import_configured_repositories(repository_store.as_ref(), &repository_config)?; + load_configured_repositories_from_store( + repository_store.as_ref(), + &repository_config, + ) + }) + .await?; let embedded_runtime = EmbeddedWorkerRuntime::new_fs_store_with_execution_backend( config.workspace_id.clone(), config.embedded_runtime_store_root.clone(), @@ -3490,20 +3667,34 @@ impl WorkspaceApi { RuntimeSubscriptionBroker::new(config.workspace_id.clone()); runtime_subscription_broker .register_embedded_runtime(embedded_runtime_id, embedded_subscription_runtime); - let config_store = Arc::new(crate::SqliteWorkspaceStore::open( - config.database_path.clone(), - )?); - let repository_secrets = Arc::new(RepositorySecretService::open( - config_store.clone(), - &config.database_path, - )?); + let database_path = config.database_path.clone(); + let executor = store.db_executor(); + let config_executor = executor.clone(); + let (config_store, repository_secrets) = executor + .run(move || { + let config_store = Arc::new( + crate::SqliteWorkspaceStore::open(database_path.clone())? + .with_db_executor(config_executor), + ); + let repository_secrets = Arc::new(RepositorySecretService::open( + config_store.clone(), + &database_path, + )?); + Ok::<_, Error>((config_store, repository_secrets)) + }) + .await?; let signing_materials: Arc = Arc::new(FsWorkspaceSigningMaterialStore::new( workspace_signing_material_root(&config.database_path), )); let signing_identities = WorkspaceSigningIdentityService::new(store.clone(), signing_materials); - signing_identities.get_validated(&config.workspace_id)?; + let identities = signing_identities.clone(); + let workspace_id = config.workspace_id.clone(); + store + .db_executor() + .run(move || identities.get_validated(&workspace_id)) + .await?; let backend_url = config .backend_base_url .clone() @@ -3546,11 +3737,19 @@ impl WorkspaceApi { .with_provider(Arc::new(RuntimeConfigSchemaProvider)) .with_provider(Arc::new(RepositoryAccessConfigSchemaProvider)) .with_provider(Arc::new(skills::SkillConfigSchemaProvider)); - config_store.ensure_workspace_config_materialized_with_schema( - &config.workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), - config_schema_registry.compose()?, - )?; + let materialization_store = config_store.clone(); + let workspace_id = config.workspace_id.clone(); + let schema = config_schema_registry.compose()?; + store + .db_executor() + .run(move || { + materialization_store.ensure_workspace_config_materialized_with_schema( + &workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + schema, + ) + }) + .await?; let worker_projection = crate::worker_projection::WorkerProjectionService::new( config.workspace_id.clone(), store.clone(), @@ -3561,15 +3760,24 @@ impl WorkspaceApi { .await .map_err(|error| Error::InvalidInput(error.to_string()))?; } - let feature_storage = crate::FeatureStorage::for_server_database(&config.database_path) - .and_then(|storage| storage.workspace(&config.workspace_id)) - .map_err(|error| { - Error::Store(format!("failed to initialize Feature storage: {error}")) - })?; - let subjektiv_registration = crate::subjektiv::SubjektivStore::register(&feature_storage) - .map_err(|error| { - Error::Store(format!("failed to register subjektiv storage: {error}")) - })?; + let database_path = config.database_path.clone(); + let workspace_id = config.workspace_id.clone(); + let (feature_storage, subjektiv_registration, authority) = store + .db_executor() + .run(move || { + let feature_storage = crate::FeatureStorage::for_server_database(&database_path) + .and_then(|storage| storage.workspace(&workspace_id)) + .map_err(|error| { + Error::Store(format!("failed to initialize Feature storage: {error}")) + })?; + let registration = crate::subjektiv::SubjektivStore::register(&feature_storage) + .map_err(|error| { + Error::Store(format!("failed to register subjektiv storage: {error}")) + })?; + let authority = SqliteWorkspaceAuthority::new(database_path, workspace_id)?; + Ok::<_, Error>((feature_storage, registration, authority)) + }) + .await?; let drive_host = Arc::new(crate::workspace_drive_host::WorkspaceDriveHost::new(&feature_storage).await?); let mut api = Self { @@ -3582,10 +3790,7 @@ impl WorkspaceApi { crate::prompt_settings::WorkspacePromptProjectionCache::default(), feature_storage, subjektiv_registration, - authority: SqliteWorkspaceAuthority::new( - config.database_path.clone(), - config.workspace_id.clone(), - )?, + authority, _worker_projection_shutdown: Arc::new(WorkerProjectionShutdownGuard { service: Arc::downgrade(&worker_projection), }), @@ -4080,77 +4285,44 @@ impl WorkspaceApi { { return false; } - let Ok(handle) = tokio::runtime::Handle::try_current() else { - tracing::warn!( - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "Backend Job Worker cleanup remains durable because no Tokio runtime is available" - ); + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + tracing::warn!(job_id = %attempt.job_id, "Backend Job cleanup remains durable without a runtime"); return false; }; - let (attempt, claimed) = match self.store.claim_backend_job_worker_cleanup( - &self.config.workspace_id, - &attempt.job_id, - &attempt.attempt_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { - Ok(claim) => claim, - Err(error) => { - tracing::warn!( - %error, - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "failed to claim durable Backend Job Worker cleanup" - ); - return false; - } - }; - if !claimed { - return false; - } - let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed - || attempt.failure_category.as_deref() == Some("invalid_result"); - if waits_for_run_teardown { - let api = self.clone(); - let cleanup_handle = handle.clone(); - handle.spawn(async move { - if let Err(error) = api.wait_for_backend_job_worker_run_teardown(&attempt).await { - tracing::warn!( - %error, - job_id = %attempt.job_id, - attempt_id = %attempt.attempt_id, - "Backend Job Worker Run teardown fence failed" - ); - return; - } - api.spawn_claimed_backend_job_worker_cleanup(&cleanup_handle, attempt); - }); - } else { - self.spawn_claimed_backend_job_worker_cleanup(&handle, attempt); - } - true - } - - fn spawn_claimed_backend_job_worker_cleanup( - &self, - handle: &tokio::runtime::Handle, - attempt: BackendJobAttemptRecord, - ) { let api = self.clone(); let job_id = attempt.job_id.clone(); let attempt_id = attempt.attempt_id.clone(); - handle.spawn_blocking(move || { + // Admit before claiming: overload must leave durable Pending/Failed + // ownership recoverable, never strand a Running claim without a job. + let admission = backend_job_cleanup_executor().and_then(|executor| { + executor.dispatch(move || { let cleanup = || -> Result<()> { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| Error::Store(error.to_string()))? - .block_on(api.cleanup_backend_job_attempt_worker(attempt)) + let (attempt, claimed) = api.store.claim_backend_job_worker_cleanup( + &api.config.workspace_id, &job_id, &attempt_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; + if !claimed { return Ok(()); } + let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed + || attempt.failure_category.as_deref() == Some("invalid_result"); + runtime.block_on(async { + if waits_for_run_teardown { + api.wait_for_backend_job_worker_run_teardown(&attempt).await?; + } + // Already on the Host owner; removal must not submit to and + // wait on its own lane. All transaction helpers return before I/O. + api.cleanup_backend_job_attempt_worker(attempt).await + }) }; if let Err(error) = cleanup() { tracing::warn!(%error, %job_id, %attempt_id, "Backend Job Worker cleanup failed"); } + }) }); + if let Err(error) = admission { + tracing::warn!(%error, "Backend Job cleanup was not admitted; its durable ownership is unchanged"); + return false; + } + true } async fn wait_for_backend_job_worker_run_teardown( @@ -4285,7 +4457,7 @@ impl WorkspaceApi { // recover that midpoint; without such an operation this remains // a diagnosable retryable failure rather than a direct DB delete. let response = WorkerRemovalService::new(self) - .execute_target_removal( + .execute_target_removal_on_host( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4351,7 +4523,7 @@ impl WorkspaceApi { } let response = WorkerRemovalService::new(self) - .execute_target_removal( + .execute_target_removal_on_host( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), @@ -4478,6 +4650,7 @@ impl WorkspaceApi { let attempt_id = attempt_id.to_string(); handle.spawn(async move { tokio::time::sleep(delay).await; + let admission = host_io_executor().and_then(|executor| executor.dispatch(move || { let Ok(Some(attempt)) = api.store .get_backend_job_attempt(&api.config.workspace_id, &job_id, &attempt_id) @@ -4501,6 +4674,10 @@ impl WorkspaceApi { { api.drain_reserved_backend_jobs(); } + })); + if let Err(error) = admission { + tracing::warn!(%error, "Backend Job timeout was not admitted; its deadline remains durable"); + } }); } @@ -6218,7 +6395,7 @@ async fn attach_server_request_context( } else { let auth = service.auth_api(); match resolve_request_actor( - auth.store.as_ref(), + auth.store.clone(), request.headers(), &auth_public_config(&auth.config).cookie_name, ) @@ -7171,7 +7348,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyRegistrationComplete, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = complete_passkey_registration(&self.auth_api(), request) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || complete_passkey_registration(&api, request)) + .await .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyRegistrationComplete::Status200 { @@ -7202,7 +7384,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthPasskeyLoginComplete, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = complete_passkey_login(&self.auth_api(), request) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || complete_passkey_login(&api, request)) + .await .map_err(ApiError::into_repository_api_error)?; Ok( server_api::server_api_responses::AuthPasskeyLoginComplete::Status200 { @@ -7235,7 +7422,11 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - approve_device_login(&self.auth_api(), actor, request) + let api = self.auth_api(); + api.store + .db_executor() + .run(move || approve_device_login(&api, actor, request)) + .await .map_err(ApiError::into_repository_api_error) } @@ -7265,7 +7456,12 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::AuthLogout, server_api::RepositoryApiError, > { - let (body, header_set_cookie) = logout_browser_session(&self.auth_api(), cookie) + let api = self.auth_api(); + let (body, header_set_cookie) = api + .store + .db_executor() + .run(move || logout_browser_session(&api, cookie)) + .await .map_err(ApiError::into_repository_api_error)?; Ok(server_api::server_api_responses::AuthLogout::Status200 { body, @@ -7280,9 +7476,12 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.server_api()?; let Some(actor) = context.actor else { + let catalog = api.catalog.clone(); return if api - .catalog - .is_empty() + .store + .db_executor() + .run(move || catalog.is_empty()) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())? { Ok(WorkspaceCatalogListResponse(Vec::new())) @@ -7295,12 +7494,17 @@ impl server_api::ServerApi for ServerApiContractService { )) }; }; + let catalog = api.catalog.clone(); let workspaces = api - .catalog - .list( - &actor.account_id, - query.limit.unwrap_or(100).try_into().unwrap_or(200), - ) + .store + .db_executor() + .run(move || { + catalog.list( + &actor.account_id, + query.limit.unwrap_or(100).try_into().unwrap_or(200), + ) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; Ok(WorkspaceCatalogListResponse( workspaces.into_iter().map(workspace_summary).collect(), @@ -7321,9 +7525,12 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; + let catalog = api.catalog.clone(); let created = api - .catalog - .create(request, actor.account_id) + .store + .db_executor() + .run(move || catalog.create(request, actor.account_id)) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; api.router_for_workspace(&created.workspace.workspace_id) .await @@ -7368,10 +7575,14 @@ impl server_api::ServerApi for ServerApiContractService { ) })?; let admission_for_workspace = api.admission_for_workspace(&workspace_id).await; - let _mutation_guard = admission_for_workspace.lock_deletion().await; + let mutation_guard = Arc::new(admission_for_workspace.lock_deletion().await); + let account_id = actor.account_id.clone(); + let operation_id = request.operation_id.clone(); let existing = api - .store - .workspace_deletion_operation(&actor.account_id, &request.operation_id) + .execute_store_stage(move |store| { + store.workspace_deletion_operation(&account_id, &operation_id) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; if existing.is_none() { let preflight = api @@ -7387,9 +7598,18 @@ impl server_api::ServerApi for ServerApiContractService { )); } } + let account_id = actor.account_id.clone(); + let id = workspace_id.clone(); + let reserve_request = request.clone(); + let reserve_guard = mutation_guard.clone(); let reservation = api - .store - .reserve_workspace_deletion(&actor.account_id, &workspace_id, &request) + .execute_store_stage(move |store| { + // Keep deletion closed until the accepted reservation really finishes, + // even when the HTTP task drops its original guard. + let _guard = reserve_guard; + store.reserve_workspace_deletion(&account_id, &id, &reserve_request) + }) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; if let Some(handle) = api.hook_handles.lock().await.remove(&workspace_id) { handle.abort(); @@ -7429,17 +7649,19 @@ impl server_api::ServerApi for ServerApiContractService { Vec::new(), ) })?; - api.store - .workspace_deletion_operation(&actor.account_id, &operation_id) - .map_err(|error| ApiError::from(error).into_repository_api_error())? - .ok_or_else(|| { - server_api::RepositoryApiError::new( - StatusCode::NOT_FOUND.as_u16(), - "Not Found", - "Workspace deletion operation not found", - Vec::new(), - ) - }) + api.execute_store_stage(move |store| { + store.workspace_deletion_operation(&actor.account_id, &operation_id) + }) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? + .ok_or_else(|| { + server_api::RepositoryApiError::new( + StatusCode::NOT_FOUND.as_u16(), + "Not Found", + "Workspace deletion operation not found", + Vec::new(), + ) + }) } async fn workspace_current( @@ -7474,13 +7696,20 @@ impl server_api::ServerApi for ServerApiContractService { server_api::server_api_responses::WorkspaceRuntimeConfig, server_api::RepositoryApiError, > { - workspace_runtime_config_result( - self.workspace_api()?, - context, - &workspace_id, - query, - if_none_match, - ) + let api = self.workspace_api()?.clone(); + api.store + .db_executor() + .run(move || { + Ok::<_, Error>(workspace_runtime_config_result( + &api, + context, + &workspace_id, + query, + if_none_match, + )) + }) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? } async fn workspace_metadata_settings( @@ -9364,7 +9593,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; restore_runtime_worker_with_context(api, runtime_id, worker_id, operation_context, request) .await .map(|Json(response)| response) @@ -9382,7 +9611,7 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; scoped_restore_runtime_worker_with_context( @@ -9409,10 +9638,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, true) @@ -9429,10 +9659,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .set_pinned(&operation_context, false) @@ -9469,7 +9700,7 @@ impl server_api::ServerApi for ServerApiContractService { server_api::RepositoryApiError, > { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; execute_runtime_cleanup_with_context(&api, &runtime_id, request, &operation_context) @@ -9486,8 +9717,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -9508,10 +9740,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; let request = worker_input_request_from_api(request) .map_err(|error| ApiError::from(error).into_repository_api_error())?; @@ -9671,8 +9904,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -9694,10 +9928,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .stop( @@ -9718,8 +9953,9 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -9741,10 +9977,11 @@ impl server_api::ServerApi for ServerApiContractService { ) -> std::result::Result { let api = self.workspace_api()?.clone(); - let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let operation_context = WorkerOperationContext::from_request_async(&api, &context).await?; validate_workspace_scope(&api, &workspace_id) .map_err(ApiError::into_repository_api_error)?; - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id) + .await .map_err(|error| ApiError::from(error).into_repository_api_error())?; worker .cancel( @@ -10559,23 +10796,28 @@ impl server_api::ServerApi for ServerApiContractService { query: server_api::TicketAssignmentTrackingQuery, ) -> std::result::Result { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let ticket_id = resolve_workspace_ticket_reference(api, &workspace_id, &ticket_id) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let values = if let Some(assignment_id) = query.assignment_id { - api.store - .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) - .map(|value| value.into_iter().collect::>()) - } else { - api.authority.ticket_assignment_tracking(&ticket_id) - } - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - Ok(server_api::TicketAssignmentTrackingList(values)) + let api = self.workspace_api()?.clone(); + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let values = if let Some(assignment_id) = query.assignment_id { + api.store + .get_ticket_assignment_tracking(&workspace_id, &ticket_id, &assignment_id) + .map(|value| value.into_iter().collect::>()) + } else { + api.authority.ticket_assignment_tracking(&ticket_id) + } + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + Ok(server_api::TicketAssignmentTrackingList(values)) + }) + .await } async fn ticket_assignment_tracking_update( @@ -11402,21 +11644,28 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirCatalogResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) + let api = self.workspace_api()?.clone(); + let headers = current_worker_contract_headers(&context)?; + let stage_api = api.clone(); + let worker = host_io_executor() + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &workspace_id, + &headers, + )?) + }) + .await .map_err(ApiError::into_repository_api_error)?; - let worker = current_worker_identity( - api, - &workspace_id, - ¤t_worker_contract_headers(&context)?, - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - { - let session_lock = current_worker_session_lock(api, &worker); - let _guard = session_lock.lock().await; - list_current_worker_workdir_catalog_for_worker(api, &worker, query) - } + let session_lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(session_lock.lock_owned().await); + run_worker_workdir_db_stage_locked(&api, &guard, move |api| { + list_current_worker_workdir_catalog_for_worker(&api, &worker, query) + }) + .await .map_err(ApiError::into_repository_api_error) } @@ -11429,20 +11678,29 @@ impl server_api::ServerApi for ServerApiContractService { server_api::CurrentWorkerWorkdirAttachmentListResponse, server_api::RepositoryApiError, > { - let api = self.workspace_api()?; - validate_workspace_scope(api, &workspace_id) + let api = self.workspace_api()?.clone(); + let headers = current_worker_contract_headers(&context)?; + let stage_api = api.clone(); + let worker = host_io_executor() + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &workspace_id, + &headers, + )?) + }) + .await .map_err(ApiError::into_repository_api_error)?; - let worker = current_worker_identity( - api, - &workspace_id, - ¤t_worker_contract_headers(&context)?, - ) - .map_err(ApiError::from) - .map_err(ApiError::into_repository_api_error)?; - let session_lock = current_worker_session_lock(api, &worker); - let _session_guard = session_lock.lock().await; - list_current_worker_workdir_attachments(api, &worker, query) - .map_err(ApiError::into_repository_api_error) + let session_lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(session_lock.lock_owned().await); + run_worker_workdir_db_stage_locked(&api, &guard, move |api| { + list_current_worker_workdir_attachments(&api, &worker, query) + }) + .await + .map_err(ApiError::into_repository_api_error) } async fn current_worker_workdir_attach( @@ -12144,8 +12402,13 @@ async fn scoped_list_flows( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult>> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.store.list_flow_sources(&path.workspace_id)?)) + }) + .await } async fn scoped_put_flow( @@ -12153,28 +12416,34 @@ async fn scoped_put_flow( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let definition = flow::compile_flow_source(&request.content).map_err(|error| { - Error::InvalidInput(format!( - "invalid Flow source: {}", - serde_json::to_string(&error.diagnostics) - .unwrap_or_else(|_| "diagnostics unavailable".to_string()) - )) - })?; - let expected_path = format!("flows/{}.dcdl", definition.name); - if request.path != expected_path { - return Err( - Error::InvalidInput(format!("Flow source path must be `{expected_path}`")).into(), - ); - } - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); - Ok(Json(api.store.put_flow_source_for_kind( - &path.workspace_id, - FlowSourceKind::Workspace, - &request.path, - &request.content, - &now, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let definition = flow::compile_flow_source(&request.content).map_err(|error| { + Error::InvalidInput(format!( + "invalid Flow source: {}", + serde_json::to_string(&error.diagnostics) + .unwrap_or_else(|_| "diagnostics unavailable".to_string()) + )) + })?; + let expected_path = format!("flows/{}.dcdl", definition.name); + if request.path != expected_path { + return Err(Error::InvalidInput(format!( + "Flow source path must be `{expected_path}`" + )) + .into()); + } + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + Ok(Json(api.store.put_flow_source_for_kind( + &path.workspace_id, + FlowSourceKind::Workspace, + &request.path, + &request.content, + &now, + )?)) + }) + .await } async fn scoped_resolve_flow_source( @@ -12182,65 +12451,79 @@ async fn scoped_resolve_flow_source( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = match &request.selector { - flow::FlowSelector::Builtin { slug } => { - let builtin = flow::builtin_flow_source(slug) - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let definition = builtin.compile().map_err(|error| { - Error::Store(format!( - "compile built-in Flow {slug:?}: {:?}", - error.diagnostics - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: format!("builtin:{slug}"), - content_digest: definition.content_digest.clone(), - definition, - } - } - flow::FlowSelector::Workspace { slug } => { - let source = api - .store - .get_flow_source_by_name(&path.workspace_id, FlowSourceKind::Workspace, slug)? - .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; - let content = api - .store - .get_flow_source_content( - &path.workspace_id, - &source.flow_id, - &source.content_digest, - )? - .ok_or_else(|| { - Error::Store(format!( - "resolved Flow content {}@{} is missing", - source.flow_id, source.content_digest - )) - })?; - ResolvedFlowSource { - selector: request.selector.clone(), - workspace_id: path.workspace_id, - flow_id: source.flow_id, - content_digest: content.content_digest, - definition: content.definition, - } - } - }; - Ok(Json(resolved)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let resolved = match &request.selector { + flow::FlowSelector::Builtin { slug } => { + let builtin = flow::builtin_flow_source(slug) + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let definition = builtin.compile().map_err(|error| { + Error::Store(format!( + "compile built-in Flow {slug:?}: {:?}", + error.diagnostics + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: format!("builtin:{slug}"), + content_digest: definition.content_digest.clone(), + definition, + } + } + flow::FlowSelector::Workspace { slug } => { + let source = api + .store + .get_flow_source_by_name( + &path.workspace_id, + FlowSourceKind::Workspace, + slug, + )? + .ok_or_else(|| Error::InvalidRecordId(request.selector.to_string()))?; + let content = api + .store + .get_flow_source_content( + &path.workspace_id, + &source.flow_id, + &source.content_digest, + )? + .ok_or_else(|| { + Error::Store(format!( + "resolved Flow content {}@{} is missing", + source.flow_id, source.content_digest + )) + })?; + ResolvedFlowSource { + selector: request.selector.clone(), + workspace_id: path.workspace_id, + flow_id: source.flow_id, + content_digest: content.content_digest, + definition: content.definition, + } + } + }; + Ok(Json(resolved)) + }) + .await } async fn scoped_get_flow( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = api - .store - .get_flow_source(&path.workspace_id, &path.flow_id)? - .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; - Ok(Json(source)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = api + .store + .get_flow_source(&path.workspace_id, &path.flow_id)? + .ok_or_else(|| Error::InvalidRecordId(path.flow_id))?; + Ok(Json(source)) + }) + .await } async fn scoped_get_workspace_settings( @@ -12311,8 +12594,13 @@ async fn scoped_get_workspace_signing_identity( "Workspace public identity access", ) .await?; - let identity = api.signing_identities.get_validated(&path.workspace_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) + api.store + .db_executor() + .run(move || { + let identity = api.signing_identities.get_validated(&path.workspace_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) + }) + .await } async fn scoped_provision_workspace_signing_identity( @@ -12327,10 +12615,15 @@ async fn scoped_provision_workspace_signing_identity( "Workspace signing identity provisioning", ) .await?; - let identity = api - .signing_identities - .provision_existing(&path.workspace_id, &actor.account_id)?; - Ok(Json(project_workspace_signing_identity(&api, identity)?)) + api.store + .db_executor() + .run(move || { + let identity = api + .signing_identities + .provision_existing(&path.workspace_id, &actor.account_id)?; + Ok(Json(project_workspace_signing_identity(&api, identity)?)) + }) + .await } fn project_workspace_signing_identity( @@ -12408,12 +12701,19 @@ async fn scoped_get_workspace_config_history( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let snapshot = api - .config_store - .load_workspace_config_history(&path.workspace_id, &path.content_digest)? - .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.content_digest.clone())))?; - Ok(Json(config_tree_snapshot_to_api(snapshot))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let snapshot = api + .config_store + .load_workspace_config_history(&path.workspace_id, &path.content_digest)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId(path.content_digest.clone())) + })?; + Ok(Json(config_tree_snapshot_to_api(snapshot))) + }) + .await } #[derive(Debug, Deserialize)] @@ -12426,34 +12726,44 @@ async fn scoped_get_prompt_projection( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidInput( - "Workspace config is not initialized".to_string(), - )) - })?; - let projection = api - .prompt_projection_cache - .resolve(&path.workspace_id, &state)?; - Ok(Json(projection.as_ref().clone())) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidInput( + "Workspace config is not initialized".to_string(), + )) + })?; + let projection = api + .prompt_projection_cache + .resolve(&path.workspace_id, &state)?; + Ok(Json(projection.as_ref().clone())) + }) + .await } async fn scoped_get_workspace_memory_settings( State(api): State, AxumPath(workspace_id): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .get_workspace_memory_settings(&workspace_id) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .get_workspace_memory_settings(&workspace_id) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) + }) + .await } async fn scoped_update_workspace_memory_settings( @@ -12461,19 +12771,24 @@ async fn scoped_update_workspace_memory_settings( AxumPath(workspace_id): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let settings = api - .config_store - .update_workspace_memory_settings( - &workspace_id, - &request.expected_language, - &request.language, - ) - .map_err(ApiError::from)?; - Ok(Json(server_api::WorkspaceMemorySettings { - workspace_id: settings.workspace_id, - language: settings.language, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let settings = api + .config_store + .update_workspace_memory_settings( + &workspace_id, + &request.expected_language, + &request.language, + ) + .map_err(ApiError::from)?; + Ok(Json(server_api::WorkspaceMemorySettings { + workspace_id: settings.workspace_id, + language: settings.language, + })) + }) + .await } async fn require_workspace_owner( @@ -12984,41 +13299,51 @@ async fn scoped_get_workspace_config_tree( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { - snapshot: ConfigTreeSnapshot::empty(), - contract: config_source::ToolchainContract::new( - config_source::DEFAULT_SCHEMA_VERSION, - Vec::new(), - config_source::DEFAULT_IMPORT_POLICY_VERSION, - ), - projection_digest: config_source::digest_bytes(b"[]"), - }); - Ok(Json(workspace_config_state_to_api(state))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .unwrap_or_else(|| crate::config_source::WorkspaceConfigState { + snapshot: ConfigTreeSnapshot::empty(), + contract: config_source::ToolchainContract::new( + config_source::DEFAULT_SCHEMA_VERSION, + Vec::new(), + config_source::DEFAULT_IMPORT_POLICY_VERSION, + ), + projection_digest: config_source::digest_bytes(b"[]"), + }); + Ok(Json(workspace_config_state_to_api(state))) + }) + .await } async fn scoped_get_workspace_config_entry( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let virtual_path = config_source::VirtualPath::parse(&path.path) - .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - let entry = state - .snapshot - .get(&virtual_path) - .cloned() - .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; - Ok(Json(config_entry_to_api(entry))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let virtual_path = config_source::VirtualPath::parse(&path.path) + .map_err(|error| ApiError::from(Error::InvalidInput(error.to_string())))?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + let entry = state + .snapshot + .get(&virtual_path) + .cloned() + .ok_or_else(|| ApiError::from(Error::InvalidRecordId(path.path)))?; + Ok(Json(config_entry_to_api(entry))) + }) + .await } async fn scoped_commit_workspace_config_tree( @@ -13026,13 +13351,18 @@ async fn scoped_commit_workspace_config_tree( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - let request = config_commit_request_from_api(request)?; - validate_workspace_scope(&api, &path.workspace_id)?; - let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; - Ok(( - StatusCode::CREATED, - Json(workspace_config_state_to_api(state)), - )) + api.store + .db_executor() + .run(move || { + let request = config_commit_request_from_api(request)?; + validate_workspace_scope(&api, &path.workspace_id)?; + let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; + Ok(( + StatusCode::CREATED, + Json(workspace_config_state_to_api(state)), + )) + }) + .await } /// Shared canonical commit and activation boundary for UI and logical config operations. @@ -13096,20 +13426,25 @@ async fn scoped_get_profile_settings( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) - })?; - Ok(Json( - crate::profile_settings::project_profiles_from_workspace_config( - &path.workspace_id, - &state, - )? - .settings, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + ApiError::from(Error::InvalidRecordId("virtual config source tree".into())) + })?; + Ok(Json( + crate::profile_settings::project_profiles_from_workspace_config( + &path.workspace_id, + &state, + )? + .settings, + )) + }) + .await } async fn scoped_list_tickets( @@ -13125,8 +13460,13 @@ async fn scoped_get_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - browser_ticket_detail(&api, &path.id) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_query_tickets( @@ -13134,8 +13474,13 @@ async fn scoped_query_tickets( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_tickets(query)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_tickets(query)?)) + }) + .await } async fn scoped_show_ticket( @@ -13143,11 +13488,16 @@ async fn scoped_show_ticket( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let backend = browser_ticket_backend(&api)?; - Ok(Json(api.authority.read_ticket_detail_with_backend( - &path.id, query, &backend, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let backend = browser_ticket_backend(&api)?; + Ok(Json(api.authority.read_ticket_detail_with_backend( + &path.id, query, &backend, + )?)) + }) + .await } fn parse_ticket_assignment_role(role: &str) -> ApiResult { @@ -13223,19 +13573,24 @@ async fn scoped_list_ticket_assignments( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let ticket = api.authority.ticket(&path.id)?; - let assignments = api - .store - .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; - Ok(Json(server_api::TicketRoleAssignmentsResponse { - workspace_id: path.workspace_id, - ticket_id: ticket.id, - assignments: assignments - .into_iter() - .map(project_ticket_role_assignment_record) - .collect(), - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let ticket = api.authority.ticket(&path.id)?; + let assignments = api + .store + .list_current_ticket_role_assignments(&path.workspace_id, &ticket.id)?; + Ok(Json(server_api::TicketRoleAssignmentsResponse { + workspace_id: path.workspace_id, + ticket_id: ticket.id, + assignments: assignments + .into_iter() + .map(project_ticket_role_assignment_record) + .collect(), + })) + }) + .await } async fn compensate_manual_ticket_worker_binding( @@ -13824,7 +14179,11 @@ async fn scoped_set_ticket_assignment_with_context( request: Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let (author, source) = authenticated_assignment_source(&api, &headers, context)?; + let source_api = api.clone(); + let context = context.clone(); + let (author, source) = host_io_executor()? + .run(move || authenticated_assignment_source(&source_api, &headers, &context)) + .await?; apply_set_ticket_assignment(State(api), path, request, author, Some(source)).await } @@ -13835,46 +14194,75 @@ async fn apply_set_ticket_assignment( assigned_by: String, source: Option, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = require_ticket_assignment_value("operation_id", request.operation_id)?; - let expected_assignment_id = request - .expected_assignment_id - .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) - .transpose()?; - let principal = internal_ticket_assignment_principal(request.principal)?; - let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let record = TicketRoleAssignmentRecord { - workspace_id: workspace_id.clone(), - ticket_id: ticket.id.clone(), - assignment_id: new_id("tasg"), - role, - principal, - assigned_by, - assigned_at, - }; + let prep_api = api.clone(); + let prep_workspace = workspace_id.clone(); + let (ticket, role, operation_id, expected_assignment_id, record, workdir_bindings) = api + .store + .db_executor() + .run(move || { + let api = &prep_api; + let workspace_id = prep_workspace; + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = + require_ticket_assignment_value("operation_id", request.operation_id)?; + let expected_assignment_id = request + .expected_assignment_id + .map(|value| require_ticket_assignment_value("expected_assignment_id", value)) + .transpose()?; + let principal = internal_ticket_assignment_principal(request.principal)?; + let assigned_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let record = TicketRoleAssignmentRecord { + workspace_id: workspace_id.clone(), + ticket_id: ticket.id.clone(), + assignment_id: new_id("tasg"), + role, + principal, + assigned_by, + assigned_at, + }; + Ok::<_, ApiError>(( + ticket, + role, + operation_id, + expected_assignment_id, + record, + request.workdir_bindings, + )) + }) + .await?; let assignment = match role { TicketAssignmentRole::Orchestrator => { - if let Some(source) = source.as_ref() { - api.store.set_current_ticket_orchestrator_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - source, - )? - } else { - // Only the cfg(test) direct-handler adapter lacks a Backend source. - api.store.set_current_ticket_role_assignment( - &record, - expected_assignment_id.as_deref(), - &new_id("tasev"), - &operation_id, - expected_assignment_id.is_some(), - )? - } + let store = api.store.clone(); + let record = record.clone(); + let expected = expected_assignment_id.clone(); + let operation_id = operation_id.clone(); + api.store + .db_executor() + .run(move || { + let expected_assignment_id = expected; + if let Some(source) = source.as_ref() { + store.set_current_ticket_orchestrator_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + source, + ) + } else { + // Only the cfg(test) direct-handler adapter lacks a Backend source. + store.set_current_ticket_role_assignment( + &record, + expected_assignment_id.as_deref(), + &new_id("tasev"), + &operation_id, + expected_assignment_id.is_some(), + ) + } + }) + .await? } TicketAssignmentRole::Worker => { if expected_assignment_id.is_some() { @@ -13884,13 +14272,8 @@ async fn apply_set_ticket_assignment( ) .into()); } - start_manual_ticket_worker_assignment( - &api, - &record, - &operation_id, - &request.workdir_bindings, - ) - .await? + start_manual_ticket_worker_assignment(&api, &record, &operation_id, &workdir_bindings) + .await? } TicketAssignmentRole::Owner | TicketAssignmentRole::Contributor => { return Err(Error::TicketAssignmentConflict( @@ -13924,7 +14307,11 @@ async fn scoped_clear_ticket_assignment_with_context( query: Query, ) -> ApiResult> { validate_workspace_scope(&api, &path.0.0)?; - let (author, source) = authenticated_assignment_source(&api, &headers, context)?; + let source_api = api.clone(); + let context = context.clone(); + let (author, source) = host_io_executor()? + .run(move || authenticated_assignment_source(&source_api, &headers, &context)) + .await?; apply_clear_ticket_assignment(State(api), path, query, author, Some(source)).await } @@ -13935,65 +14322,70 @@ async fn apply_clear_ticket_assignment( author: String, source: Option, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket = api.authority.ticket(&id)?; - let role = parse_ticket_assignment_role(&role)?; - let operation_id = query - .operation_id - .map(|value| require_ticket_assignment_value("operation_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) - })?; - let assignment_id = query - .assignment_id - .map(|value| require_ticket_assignment_value("assignment_id", value)) - .transpose()? - .ok_or_else(|| { - Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) - })?; - let event_id = new_id("tasev"); - let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - let reason = Some("role assignment removed from Ticket detail"); - let cleared = if role == TicketAssignmentRole::Orchestrator - && let Some(source) = source.as_ref() - { - api.store.clear_current_ticket_orchestrator_assignment( - &workspace_id, - &ticket.id, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - source, - )? - } else { - api.store.clear_current_ticket_role_assignment( - &workspace_id, - &ticket.id, - role, - &assignment_id, - &event_id, - &operation_id, - &author, - &occurred_at, - reason, - )? - }; - if !cleared { - return Err(Error::TicketAssignmentConflict(format!( - "assignment `{assignment_id}` is not current for role `{}`", - role.as_str() - )) - .into()); - } - Ok(Json(server_api::TicketRoleAssignmentMutationResponse { - workspace_id, - ticket_id: ticket.id, - assignment: None, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let ticket = api.authority.ticket(&id)?; + let role = parse_ticket_assignment_role(&role)?; + let operation_id = query + .operation_id + .map(|value| require_ticket_assignment_value("operation_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires operation_id".to_string()) + })?; + let assignment_id = query + .assignment_id + .map(|value| require_ticket_assignment_value("assignment_id", value)) + .transpose()? + .ok_or_else(|| { + Error::TicketAssignmentConflict("unassign requires assignment_id".to_string()) + })?; + let event_id = new_id("tasev"); + let occurred_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let reason = Some("role assignment removed from Ticket detail"); + let cleared = if role == TicketAssignmentRole::Orchestrator + && let Some(source) = source.as_ref() + { + api.store.clear_current_ticket_orchestrator_assignment( + &workspace_id, + &ticket.id, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + source, + )? + } else { + api.store.clear_current_ticket_role_assignment( + &workspace_id, + &ticket.id, + role, + &assignment_id, + &event_id, + &operation_id, + &author, + &occurred_at, + reason, + )? + }; + if !cleared { + return Err(Error::TicketAssignmentConflict(format!( + "assignment `{assignment_id}` is not current for role `{}`", + role.as_str() + )) + .into()); + } + Ok(Json(server_api::TicketRoleAssignmentMutationResponse { + workspace_id, + ticket_id: ticket.id, + assignment: None, + })) + }) + .await } async fn scoped_cancel_ticket_implementation( @@ -14405,49 +14797,54 @@ async fn scoped_edit_ticket_item( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let mut targets = request.target.map(|target| match target { - server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { - targets: targets - .into_iter() - .map(|target| ticket::TicketTarget { - repository_key: target.repository_key, - ref_selector: target.ref_selector, - access: target.access, - }) - .collect(), - }, - server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, - }); - if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { - validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; - } - browser_ticket_backend(&api)? - .edit_item( - TicketIdOrSlug::Id(path.id.clone()), - TicketItemEdit { - title: request.title, - body: request.body.map(MarkdownText::new), - body_replacement: match (request.old_string, request.new_string) { - (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { - old_string, - new_string, - replace_all: request.replace_all, - }), - (None, None) => None, - _ => { - return Err(settings_bad_request( - "invalid_ticket_edit_replacement", - "old_string and new_string must be provided together", - )); - } + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let mut targets = request.target.map(|target| match target { + server_api::BrowserTicketTargetEdit::Set { targets } => TicketTargetsEdit::Set { + targets: targets + .into_iter() + .map(|target| ticket::TicketTarget { + repository_key: target.repository_key, + ref_selector: target.ref_selector, + access: target.access, + }) + .collect(), }, - targets, - author: request.author, - }, - ) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) + server_api::BrowserTicketTargetEdit::Clear => TicketTargetsEdit::Clear, + }); + if let Some(TicketTargetsEdit::Set { targets }) = targets.as_mut() { + validate_ticket_target_repository_keys(&api, &path.workspace_id, targets)?; + } + browser_ticket_backend(&api)? + .edit_item( + TicketIdOrSlug::Id(path.id.clone()), + TicketItemEdit { + title: request.title, + body: request.body.map(MarkdownText::new), + body_replacement: match (request.old_string, request.new_string) { + (Some(old_string), Some(new_string)) => Some(TicketBodyReplacement { + old_string, + new_string, + replace_all: request.replace_all, + }), + (None, None) => None, + _ => { + return Err(settings_bad_request( + "invalid_ticket_edit_replacement", + "old_string and new_string must be provided together", + )); + } + }, + targets, + author: request.author, + }, + ) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_append_ticket_event( @@ -14455,21 +14852,26 @@ async fn scoped_append_ticket_event( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = match request.role { - server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, - server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, - server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, - server_api::BrowserTicketThreadRole::ImplementationReport => { - TicketEventKind::ImplementationReport - } - }; - let mut event = NewTicketEvent::new(role, request.body); - event.author = request.author; - browser_ticket_backend(&api)? - .add_event(TicketIdOrSlug::Id(path.id.clone()), event) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let role = match request.role { + server_api::BrowserTicketThreadRole::Comment => TicketEventKind::Comment, + server_api::BrowserTicketThreadRole::Plan => TicketEventKind::Plan, + server_api::BrowserTicketThreadRole::Decision => TicketEventKind::Decision, + server_api::BrowserTicketThreadRole::ImplementationReport => { + TicketEventKind::ImplementationReport + } + }; + let mut event = NewTicketEvent::new(role, request.body); + event.author = request.author; + browser_ticket_backend(&api)? + .add_event(TicketIdOrSlug::Id(path.id.clone()), event) + .map_err(Error::from)?; + browser_ticket_detail(&api, &path.id) + }) + .await } async fn scoped_mark_ticket_ready_from_browser( @@ -14648,17 +15050,24 @@ fn schedule_ticket_item_check( } }; let api = api.clone(); - tokio::task::spawn_blocking(move || { - if let Err(error) = api.dispatch_backend_job_reservation(reservation) { - tracing::warn!( - job_id = %request.job_id, - ticket_input_digest = ?request.input_digest(), - %error, - "asynchronous Ticket item checker dispatch failed" - ); - api.drain_reserved_backend_jobs(); - } + let admission = host_io_executor().and_then(|executor| { + executor.dispatch(move || { + if let Err(error) = api.dispatch_backend_job_reservation(reservation) { + tracing::warn!( + job_id = %request.job_id, + ticket_input_digest = ?request.input_digest(), + %error, + "asynchronous Ticket item checker dispatch failed" + ); + if !matches!(error, Error::DbOperationOutcomeUnknown) { + api.drain_reserved_backend_jobs(); + } + } + }) }); + if let Err(error) = admission { + tracing::warn!(%error, "Ticket checker was not admitted; its reservation remains durable"); + } } async fn execute_ticket_rest_operation( @@ -14694,6 +15103,13 @@ async fn execute_ticket_rest_operation_with_context( .await?; return Ok(TicketBackendOperationResult::Ticket(updated)); } + let job_api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let context = context.cloned(); + host_io_executor()?.run(move || { + let api = &job_api; + let workspace_id = workspace_id.as_str(); + let context = context.as_ref(); validate_ticket_operation_repository_keys(api, workspace_id, &operation)?; let mut backend = SqliteTicketBackend::open_verified( api.config.database_path.clone(), @@ -14891,6 +15307,7 @@ async fn execute_ticket_rest_operation_with_context( } } Ok(result) + }).await } #[cfg(test)] @@ -15853,6 +16270,8 @@ async fn scoped_list_merge_requests( AxumPath(workspace_id): AxumPath, Query(query): Query, ) -> ApiResult> { + host_io_executor()?.run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; require_workspace_access(&workspace_id, &api)?; let ticket_id = query @@ -15961,6 +16380,8 @@ async fn scoped_list_merge_requests( items, next_cursor: page.next_cursor, })) + + }).await } fn merge_ref_diagnostic(error: ApiError) -> MergeRequestRefDiagnostic { @@ -16066,159 +16487,167 @@ async fn scoped_show_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let store = merge_request_store(&api, &workspace_id)?; - let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let integrated_refs = integrated_merge_refs(&mr); - mr.thread = store.thread_page_by_id( - &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - let ticket_id = mr.ticket_ids.first().ok_or_else(|| { - Error::MergeRequest(merge_request::MergeRequestError::Conflict( - "Merge Request has no linked Ticket".into(), - )) - })?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; - let (source, target) = if let Some(refs) = integrated_refs { - refs - } else { - let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { - (Some(selector), Some(assignment)) => { - match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - } - } - (Some(_), None) => unknown_merge_ref( - "source_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the source ref", - ), - (None, _) => server_api::MergeRequestRefResponse { - status: "requires_repair".into(), - resolved_ref: None, - observed_at: Utc::now().to_rfc3339(), - diagnostic: None, - }, - }; - let target = match assignment.as_ref() { - Some(assignment) => match observe_published_merge_ref( - &api, + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let store = merge_request_store(&api, &workspace_id)?; + let mut mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let integrated_refs = integrated_merge_refs(&mr); + mr.thread = store.thread_page_by_id( &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - ) { - Ok(observation) => merge_ref_response(observation), - Err(error) => unknown_merge_ref_response(error), - }, - None => unknown_merge_ref( - "target_ref_runtime_unavailable", - "No current Coder Runtime is available to observe the target ref", - ), - }; - (source, target) - }; - let linked_tickets = mr - .ticket_ids - .iter() - .map(|ticket_id| { - Ok(server_api::MergeRequestLinkedTicketResponse { - ticket_id: ticket_id.clone(), - key: api.store.resource_key( - &workspace_id, - WorkspaceResourceKind::Ticket, - ticket_id, - )?, - }) + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + let ticket_id = mr.ticket_ids.first().ok_or_else(|| { + Error::MergeRequest(merge_request::MergeRequestError::Conflict( + "Merge Request has no linked Ticket".into(), + )) + })?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, ticket_id)?; + let (source, target) = if let Some(refs) = integrated_refs { + refs + } else { + let source = match (mr.selector_from.as_deref(), assignment.as_ref()) { + (Some(selector), Some(assignment)) => { + match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + } + } + (Some(_), None) => unknown_merge_ref( + "source_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the source ref", + ), + (None, _) => server_api::MergeRequestRefResponse { + status: "requires_repair".into(), + resolved_ref: None, + observed_at: Utc::now().to_rfc3339(), + diagnostic: None, + }, + }; + let target = match assignment.as_ref() { + Some(assignment) => match observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + ) { + Ok(observation) => merge_ref_response(observation), + Err(error) => unknown_merge_ref_response(error), + }, + None => unknown_merge_ref( + "target_ref_runtime_unavailable", + "No current Coder Runtime is available to observe the target ref", + ), + }; + (source, target) + }; + let linked_tickets = mr + .ticket_ids + .iter() + .map(|ticket_id| { + Ok(server_api::MergeRequestLinkedTicketResponse { + ticket_id: ticket_id.clone(), + key: api.store.resource_key( + &workspace_id, + WorkspaceResourceKind::Ticket, + ticket_id, + )?, + }) + }) + .collect::>>()?; + let merge_request = public_merge_request(&api, &workspace_id, mr)?; + Ok(Json(server_api::MergeRequestDetailResponse { + merge_request, + source, + target, + linked_tickets, + })) }) - .collect::>>()?; - let merge_request = public_merge_request(&api, &workspace_id, mr)?; - Ok(Json(server_api::MergeRequestDetailResponse { - merge_request, - source, - target, - linked_tickets, - })) + .await } async fn scoped_merge_request_readiness( State(api): State, AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; - let (current_subject_ref, source_blocker) = - if mr.state == merge_request::MergeRequestState::Merged { - ( - mr.merged_result() - .ok() - .map(|merge| merge.approved_source_ref.clone()), - None, - ) - } else { - match (mr.selector_from.as_deref(), assignment) { - (Some(selector), Some(assignment)) => match observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - ) { - Ok(observation) => (Some(observation.resolved_ref), None), - Err(error) => { - let diagnostic = merge_ref_diagnostic(error); - let blocker = source_ref_readiness_blocker(&diagnostic.code); - (None, Some(blocker.to_string())) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)?; + let (current_subject_ref, source_blocker) = + if mr.state == merge_request::MergeRequestState::Merged { + ( + mr.merged_result() + .ok() + .map(|merge| merge.approved_source_ref.clone()), + None, + ) + } else { + match (mr.selector_from.as_deref(), assignment) { + (Some(selector), Some(assignment)) => match observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + ) { + Ok(observation) => (Some(observation.resolved_ref), None), + Err(error) => { + let diagnostic = merge_ref_diagnostic(error); + let blocker = source_ref_readiness_blocker(&diagnostic.code); + (None, Some(blocker.to_string())) + } + }, + (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), + (None, _) => (None, None), } + }; + let mut report = store.readiness(merge_request::ReadinessCheck { + merge_request_id, + ticket_id, + current_subject_ref, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: String::new(), + worker_id: String::new(), + assignment_id: String::new(), }, - (Some(_), None) => (None, Some("source_ref_unavailable".to_string())), - (None, _) => (None, None), + })?; + if let Some(blocker) = source_blocker { + report + .blockers + .retain(|current| current != "selector_unresolved"); + if !report.blockers.contains(&blocker) { + report.blockers.push(blocker); + } + report.ready = false; } - }; - let mut report = store.readiness(merge_request::ReadinessCheck { - merge_request_id, - ticket_id, - current_subject_ref, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: String::new(), - worker_id: String::new(), - assignment_id: String::new(), - }, - })?; - if let Some(blocker) = source_blocker { - report - .blockers - .retain(|current| current != "selector_unresolved"); - if !report.blockers.contains(&blocker) { - report.blockers.push(blocker); - } - report.ready = false; - } - Ok(Json(server_api::MergeRequestReadinessResponse { - ready: report.ready, - blockers: report.blockers, - subject_ref: report.subject_ref, - review: report.review.map(public_review_event), - })) + Ok(Json(server_api::MergeRequestReadinessResponse { + ready: report.ready, + blockers: report.blockers, + subject_ref: report.subject_ref, + review: report.review.map(public_review_event), + })) + }) + .await } async fn scoped_open_merge_request( @@ -16227,71 +16656,78 @@ async fn scoped_open_merge_request( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let repository = api.require_configured_workspace_repository_by_key(&input.repository_key)?; - let repository_id = repository.id; - require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_from, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &repository_id, - &input.selector_from, - &source_observation.resolved_ref, - )?; - observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &repository_id, - &input.selector_to, - )?; - let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( - merge_request::OpenMergeRequest { - merge_request_id: Uuid::now_v7().to_string(), - ticket_id, - repository_id: repository_id.clone(), - selector_from: input.selector_from, - selector_to: input.selector_to, - summary: input.summary, - auth: merge_request::MergeRequestAuth { - workspace_id: workspace_id.clone(), - repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }, - )?; - Ok(Json(public_merge_request( - &api, - &workspace_id, - merge_request, - )?)) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let repository = + api.require_configured_workspace_repository_by_key(&input.repository_key)?; + let repository_id = repository.id; + require_ticket_read_write_target(&api, &ticket_id, &repository_id, &input.selector_to)?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_from, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &repository_id, + &input.selector_from, + &source_observation.resolved_ref, + )?; + observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &repository_id, + &input.selector_to, + )?; + let merge_request = merge_request_store(&api, &workspace_id)?.open_merge_request( + merge_request::OpenMergeRequest { + merge_request_id: Uuid::now_v7().to_string(), + ticket_id, + repository_id: repository_id.clone(), + selector_from: input.selector_from, + selector_to: input.selector_to, + summary: input.summary, + auth: merge_request::MergeRequestAuth { + workspace_id: workspace_id.clone(), + repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }, + )?; + Ok(Json(public_merge_request( + &api, + &workspace_id, + merge_request, + )?)) + }) + .await } async fn scoped_merge_request_thread( @@ -16299,19 +16735,23 @@ async fn scoped_merge_request_thread( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult>> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( - &workspace_id, - &merge_request_id, - query.after, - query.limit.unwrap_or(100), - )?; - Ok(Json( - events - .into_iter() - .map(public_merge_request_thread_event) - .collect(), - )) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let events = merge_request_store(&api, &workspace_id)?.thread_page_by_id( + &workspace_id, + &merge_request_id, + query.after, + query.limit.unwrap_or(100), + )?; + Ok(Json( + events + .into_iter() + .map(public_merge_request_thread_event) + .collect(), + )) + }) + .await } async fn scoped_repair_merge_request_selector( @@ -16366,20 +16806,24 @@ async fn scoped_register_reviewer_child_session( AxumPath(workspace_id): AxumPath, Json(input): Json, ) -> ApiResult { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( - merge_request::RegisterReviewerChildSession { - workspace_id, - parent_runtime_id: source.runtime_id, - parent_worker_id: source.worker_id, - child_session_id: input.child_session_id, - reviewer_profile: "builtin:reviewer".into(), - now: Utc::now(), - }, - )?; - Ok(StatusCode::NO_CONTENT) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( + merge_request::RegisterReviewerChildSession { + workspace_id, + parent_runtime_id: source.runtime_id, + parent_worker_id: source.worker_id, + child_session_id: input.child_session_id, + reviewer_profile: "builtin:reviewer".into(), + now: Utc::now(), + }, + )?; + Ok(StatusCode::NO_CONTENT) + }) + .await } async fn scoped_register_merge_request_review_capability( @@ -16388,107 +16832,114 @@ async fn scoped_register_merge_request_review_capability( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let requested_ticket_id = - resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; - if requested_ticket_id != ticket_id { - return Err(Error::TicketAssignmentConflict( - "Reviewer handoff Ticket does not match the addressed Merge Request".into(), - ) - .into()); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - let source_observation = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )?; - require_assigned_workdir_source( - &api, - &assignment, - &mr.repository_id, - selector, - &source_observation.resolved_ref, - )?; - source_observation.resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err( - Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), - ); - } - }; - let mut ticket_merge_request_subjects = Vec::new(); - for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { - let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged { - merged_result_source_ref(&linked)? - } else { - let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { - Error::InvalidInput(format!( - "Merge Request `{}` selector_from requires repair", - linked.merge_request_id - )) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let requested_ticket_id = + resolve_workspace_ticket_reference(&api, &workspace_id, &input.ticket_id)?; + if requested_ticket_id != ticket_id { + return Err(Error::TicketAssignmentConflict( + "Reviewer handoff Ticket does not match the addressed Merge Request".into(), + ) + .into()); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput("selector_from requires repair".into()) + })?; + let source_observation = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )?; + require_assigned_workdir_source( + &api, + &assignment, + &mr.repository_id, + selector, + &source_observation.resolved_ref, + )?; + source_observation.resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err(Error::InvalidInput( + "closed Merge Request is not reviewable".into(), + ) + .into()); + } + }; + let mut ticket_merge_request_subjects = Vec::new(); + for linked in store.list_for_ticket(&workspace_id, &ticket_id)? { + let linked_subject_ref = if linked.state == merge_request::MergeRequestState::Merged + { + merged_result_source_ref(&linked)? + } else { + let linked_selector = linked.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput(format!( + "Merge Request `{}` selector_from requires repair", + linked.merge_request_id + )) + })?; + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &linked.repository_id, + linked_selector, + )? + .resolved_ref + }; + ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { + merge_request_id: linked.merge_request_id, + subject_ref: linked_subject_ref, + }); + } + let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; + store.request_review(merge_request::RequestMergeRequestReview { + merge_request_id, + ticket_id, + ticket_content_digest, + ticket_merge_request_subjects, + subject_ref, + child_session_id: input.child_session_id, + capability_token: input.capability_token, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), })?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &linked.repository_id, - linked_selector, - )? - .resolved_ref - }; - ticket_merge_request_subjects.push(merge_request::MergeRequestReviewSubject { - merge_request_id: linked.merge_request_id, - subject_ref: linked_subject_ref, - }); - } - let ticket_content_digest = api.authority.ticket(&ticket_id)?.content_digest; - store.request_review(merge_request::RequestMergeRequestReview { - merge_request_id, - ticket_id, - ticket_content_digest, - ticket_merge_request_subjects, - subject_ref, - child_session_id: input.child_session_id, - capability_token: input.capability_token, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - })?; - Ok(StatusCode::NO_CONTENT) + Ok(StatusCode::NO_CONTENT) + }) + .await } async fn scoped_submit_merge_request_review( @@ -16496,67 +16947,73 @@ async fn scoped_submit_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - let store = merge_request_store(&api, &workspace_id)?; - let review_authorization = - store.authorize_review_submission(&merge_request_id, &input.capability_token)?; - if review_authorization.workspace_id != workspace_id - || review_authorization.merge_request_id != merge_request_id - { - return Err( - Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( - "review grant invalid".into(), - )) - .into(), - ); - } - let ticket_id = review_authorization.ticket_id; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - let current_subject_ref = match mr.state { - merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, - merge_request::MergeRequestState::Open => { - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref - } - merge_request::MergeRequestState::Closed => { - return Err( - Error::InvalidInput("closed Merge Request is not reviewable".into()).into(), - ); - } - }; - let decision = internal_review_decision(input.decision); - let findings = input - .findings - .into_iter() - .map(internal_review_finding) - .collect(); - let event = store.submit_review(merge_request::SubmitMergeRequestReview { - merge_request_id, - ticket_id, - current_subject_ref, - capability_token: input.capability_token, - decision, - body: input.body, - findings, - now: Utc::now(), - })?; - Ok(Json(public_review_event(event))) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let review_authorization = + store.authorize_review_submission(&merge_request_id, &input.capability_token)?; + if review_authorization.workspace_id != workspace_id + || review_authorization.merge_request_id != merge_request_id + { + return Err( + Error::MergeRequest(merge_request::MergeRequestError::Unauthorized( + "review grant invalid".into(), + )) + .into(), + ); + } + let ticket_id = review_authorization.ticket_id; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + let current_subject_ref = match mr.state { + merge_request::MergeRequestState::Merged => merged_result_source_ref(&mr)?, + merge_request::MergeRequestState::Open => { + let selector = mr.selector_from.as_deref().ok_or_else(|| { + Error::InvalidInput("selector_from requires repair".into()) + })?; + observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref + } + merge_request::MergeRequestState::Closed => { + return Err(Error::InvalidInput( + "closed Merge Request is not reviewable".into(), + ) + .into()); + } + }; + let decision = internal_review_decision(input.decision); + let findings = input + .findings + .into_iter() + .map(internal_review_finding) + .collect(); + let event = store.submit_review(merge_request::SubmitMergeRequestReview { + merge_request_id, + ticket_id, + current_subject_ref, + capability_token: input.capability_token, + decision, + body: input.body, + findings, + now: Utc::now(), + })?; + Ok(Json(public_review_event(event))) + }) + .await } async fn scoped_revoke_merge_request_review( @@ -16565,44 +17022,50 @@ async fn scoped_revoke_merge_request_review( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - if !input.explicit_confirmation { - return Err(Error::BrowserReopenConfirmationRequired.into()); - } - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - if assignment.worker.runtime_id != source.runtime_id - || assignment.worker.worker_id != source.worker_id - { - return Err(Error::TicketAssignmentConflict( - "authenticated Worker is not current assignee".into(), - ) - .into()); - } - let event = store.revoke_review(merge_request::RevokeMergeRequestReview { - merge_request_id, - ticket_id, - review_event_id: input.review_event_id, - reason: input.reason, - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id, - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - })?; - Ok(Json(public_review_revoked_event(event))) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + if !input.explicit_confirmation { + return Err(Error::BrowserReopenConfirmationRequired.into()); + } + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not current assignee".into(), + ) + .into()); + } + let event = store.revoke_review(merge_request::RevokeMergeRequestReview { + merge_request_id, + ticket_id, + review_event_id: input.review_event_id, + reason: input.reason, + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id, + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + })?; + Ok(Json(public_review_revoked_event(event))) + }) + .await } async fn scoped_complete_merge_request( @@ -16611,94 +17074,100 @@ async fn scoped_complete_merge_request( AxumPath((workspace_id, merge_request_id)): AxumPath<(String, String)>, Json(input): Json, ) -> ApiResult> { - let workspace_id = parse_workspace_id(&workspace_id)?; - require_workspace_access(&workspace_id, &api)?; - let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; - require_online_workspace_orchestrator_source(&api, &source)?; - let store = merge_request_store(&api, &workspace_id)?; - let mr = store.get_by_id(&workspace_id, &merge_request_id)?; - let ticket_id = merge_request_ticket_id(&mr)?; - if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { - let replay = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: existing.approved_source_ref.clone(), - target_ref_before: input.target_ref_before, - target_ref_after: input.target_ref_after, - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: String::new(), - }, - now: Utc::now(), - }; - return store - .complete(replay) - .map(public_merge_event) - .map(Json) - .map_err(Into::into); - } - let assignment = api - .store - .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? - .ok_or_else(|| { - Error::TicketAssignmentConflict("Ticket has no active Coder work assignment".into()) - })?; - let selector = mr - .selector_from - .as_deref() - .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; - let current_source_ref = observe_published_source_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - selector, - )? - .resolved_ref; - let observed_target = observe_published_merge_ref( - &api, - &workspace_id, - &assignment.worker.runtime_id, - &mr.repository_id, - &mr.selector_to, - )?; - require_completed_target_observation( - &observed_target.resolved_ref, - &input.target_ref_before, - &input.target_ref_after, - )?; - let completion = merge_request::CompleteMergeRequest { - merge_request_id, - ticket_id, - operation_id: input.operation_id, - approval_event_id: input.approval_event_id, - current_subject_ref: current_source_ref, - target_ref_before: input.target_ref_before.clone(), - target_ref_after: input.target_ref_after.clone(), - strategy: internal_merge_strategy(input.strategy), - resolution: internal_conflict_resolution(input.resolution), - auth: merge_request::MergeRequestAuth { - workspace_id, - repository_id: mr.repository_id.clone(), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - assignment_id: assignment.assignment_id, - }, - now: Utc::now(), - }; - store.validate_completion(&completion)?; - store - .complete(completion) - .map(public_merge_event) - .map(Json) - .map_err(Into::into) + host_io_executor()? + .run(move || { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + require_online_workspace_orchestrator_source(&api, &source)?; + let store = merge_request_store(&api, &workspace_id)?; + let mr = store.get_by_id(&workspace_id, &merge_request_id)?; + let ticket_id = merge_request_ticket_id(&mr)?; + if let Some(existing) = recorded_merge_completion(&mr.thread, &input.operation_id) { + let replay = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: existing.approved_source_ref.clone(), + target_ref_before: input.target_ref_before, + target_ref_after: input.target_ref_after, + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: String::new(), + }, + now: Utc::now(), + }; + return store + .complete(replay) + .map(public_merge_event) + .map(Json) + .map_err(Into::into); + } + let assignment = api + .store + .get_active_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict( + "Ticket has no active Coder work assignment".into(), + ) + })?; + let selector = mr + .selector_from + .as_deref() + .ok_or_else(|| Error::InvalidInput("selector_from requires repair".into()))?; + let current_source_ref = observe_published_source_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + selector, + )? + .resolved_ref; + let observed_target = observe_published_merge_ref( + &api, + &workspace_id, + &assignment.worker.runtime_id, + &mr.repository_id, + &mr.selector_to, + )?; + require_completed_target_observation( + &observed_target.resolved_ref, + &input.target_ref_before, + &input.target_ref_after, + )?; + let completion = merge_request::CompleteMergeRequest { + merge_request_id, + ticket_id, + operation_id: input.operation_id, + approval_event_id: input.approval_event_id, + current_subject_ref: current_source_ref, + target_ref_before: input.target_ref_before.clone(), + target_ref_after: input.target_ref_after.clone(), + strategy: internal_merge_strategy(input.strategy), + resolution: internal_conflict_resolution(input.resolution), + auth: merge_request::MergeRequestAuth { + workspace_id, + repository_id: mr.repository_id.clone(), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + assignment_id: assignment.assignment_id, + }, + now: Utc::now(), + }; + store.validate_completion(&completion)?; + store + .complete(completion) + .map(public_merge_event) + .map(Json) + .map_err(Into::into) + }) + .await } // Header-only entry point is retained solely for explicit legacy internal tests. @@ -16721,47 +17190,56 @@ async fn scoped_ticket_assignment_tracking_update_with_context( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(update): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let source = authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { - Error::WorkspacePermissionDenied( - "Tracking is self-reported by a verified Runtime Worker".into(), - ) - })?; - // Tracking authority is retained responsibility plus the registered singleton, - // NOT Ticket openness, active work, Worker state or a model-supplied role. - if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { - return Err(Error::WorkspacePermissionDenied( - "Tracking updates require the registered current Workspace Orchestrator".into(), - ) - .into()); - } - let authentication_binding_id = context - .runtime_source - .as_ref() - .map(|source| source.authentication_binding_id.clone()) - .ok_or_else(|| { - Error::WorkspacePermissionDenied( + let context = context.clone(); + api.store + .db_executor() + .run(move || { + let context = &context; + + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let source = + authenticated_ticket_worker_source(&headers, context)?.ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Tracking is self-reported by a verified Runtime Worker".into(), + ) + })?; + // Tracking authority is retained responsibility plus the registered singleton, + // NOT Ticket openness, active work, Worker state or a model-supplied role. + if !find_workspace_orchestrator(&api).is_some_and(|worker| worker.worker == source) { + return Err(Error::WorkspacePermissionDenied( + "Tracking updates require the registered current Workspace Orchestrator".into(), + ) + .into()); + } + let authentication_binding_id = context + .runtime_source + .as_ref() + .map(|source| source.authentication_binding_id.clone()) + .ok_or_else(|| { + Error::WorkspacePermissionDenied( "Tracking requires the Runtime binding actually used to authenticate the request" .into(), ) - })?; - let source_context = ticket::TicketAssignmentTrackingSourceContext { - actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), - runtime_id: source.runtime_id, - worker_id: source.worker_id, - agent_key: "workspace-orchestrator".into(), - authentication_binding_id, - }; - // The Ticket-owned mutation checks current assignment/CAS and exact replay in - // one transaction. A historical receipt never restores operational grants. - let value = api.store.update_ticket_assignment_tracking( - &workspace_id, - &ticket_id, - &update, - &source_context, - )?; - Ok(Json(value)) + })?; + let source_context = ticket::TicketAssignmentTrackingSourceContext { + actor: format!("worker:{}/{}", source.runtime_id, source.worker_id), + runtime_id: source.runtime_id, + worker_id: source.worker_id, + agent_key: "workspace-orchestrator".into(), + authentication_binding_id, + }; + // The Ticket-owned mutation checks current assignment/CAS and exact replay in + // one transaction. A historical receipt never restores operational grants. + let value = api.store.update_ticket_assignment_tracking( + &workspace_id, + &ticket_id, + &update, + &source_context, + )?; + Ok(Json(value)) + }) + .await } async fn scoped_ticket_assignment_tracking_history( @@ -16769,25 +17247,31 @@ async fn scoped_ticket_assignment_tracking_history( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; - let limit = query.limit.unwrap_or(50); - if !(1..=100).contains(&limit) { - return Err( - Error::InvalidInput("tracking history limit must be between 1 and 100".into()).into(), - ); - } - let page = api.store.page_ticket_assignment_tracking_history( - &workspace_id, - &ticket_id, - &query.assignment_id, - query.after_event_id.as_deref(), - limit, - )?; - Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { - items: page.items, - next_event_id: page.next_event_id, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &workspace_id)?; + let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; + let limit = query.limit.unwrap_or(50); + if !(1..=100).contains(&limit) { + return Err(Error::InvalidInput( + "tracking history limit must be between 1 and 100".into(), + ) + .into()); + } + let page = api.store.page_ticket_assignment_tracking_history( + &workspace_id, + &ticket_id, + &query.assignment_id, + query.after_event_id.as_deref(), + limit, + )?; + Ok(Json(server_api::TicketAssignmentTrackingHistoryPage { + items: page.items, + next_event_id: page.next_event_id, + })) + }) + .await } async fn scoped_ticket_case_update_with_context( @@ -16856,6 +17340,10 @@ async fn apply_ticket_case_update( AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, Json(mut request): Json, ) -> ApiResult> { + let actor = actor.cloned(); + api.store.db_executor().run(move || { + let actor = actor.as_ref(); + validate_workspace_scope(&api, &workspace_id)?; let ticket_id = resolve_workspace_ticket_reference(&api, &workspace_id, &ticket_id)?; let mut backend = browser_ticket_backend(&api)?; @@ -16912,6 +17400,8 @@ async fn apply_ticket_case_update( .update_case(&ticket_id, request) .map_err(Error::from)?; Ok(Json(ticket)) + + }).await } fn reject_non_browser_reopen_auth(headers: &HeaderMap) -> Result<()> { @@ -17533,44 +18023,93 @@ fn runtime_local_owner_worker_id<'a>( (caller.runtime_id == target_runtime_id).then_some(caller.worker_id.as_str()) } +// Locked async workflows stay on Tokio. Each accepted synchronous mutation +// retains this same owned guard through execution and discarded-result cleanup. +async fn run_worker_workdir_db_stage_locked( + api: &WorkspaceApi, + session_guard: &Arc>, + operation: impl FnOnce(WorkspaceApi) -> std::result::Result + Send + 'static, +) -> std::result::Result +where + T: Send + 'static, + E: From + Send + 'static, +{ + let stage_api = api.clone(); + let stage_guard = session_guard.clone(); + let (outcome, _session_guard) = api + .store + .db_executor() + .run(move || { + let outcome = operation(stage_api); + // Keep the guard with the outcome, including errors. If the waiter was + // cancelled, the job drops the outcome before releasing this guard. + Ok::<_, E>((outcome, stage_guard)) + }) + .await?; + outcome +} + async fn open_current_worker_workdir_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, ) -> Result { - let workdir = api + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_link = link.clone(); + let (workdir, effective_capabilities) = api .store - .list_workdir_registry(&api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == link.workdir_id) - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!( - "attached Workdir {} is not registered in this Workspace", - link.workdir_id - ), - })?; - if matches!( - workdir.source, - WorkdirRegistrySource::WorkspaceConfig { .. } - ) { - return Err(Error::WorkspacePermissionDenied( - "Logical config has no filesystem session".into(), - )); - } - let effective_capabilities = - effective_worker_workdir_capabilities(api, worker, &workdir, link.capabilities)?; + .db_executor() + .run(move || { + let workdir = stage_api + .store + .list_workdir_registry(&stage_api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == stage_link.workdir_id) + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: stage_worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!( + "attached Workdir {} is not registered in this Workspace", + stage_link.workdir_id + ), + })?; + if matches!( + workdir.source, + WorkdirRegistrySource::WorkspaceConfig { .. } + ) { + return Err(Error::WorkspacePermissionDenied( + "Logical config has no filesystem session".into(), + )); + } + let capabilities = effective_worker_workdir_capabilities( + &stage_api, + &stage_worker, + &workdir, + stage_link.capabilities, + )?; + Ok((workdir, capabilities)) + }) + .await?; if let WorkdirRegistrySource::ExternalGrant { grant_id } = &workdir.source { close_current_worker_attachment_session_locked(api, worker).await?; + let stage_api = api.clone(); + let stage_grant_id = grant_id.clone(); let mut grant = api .store - .get_external_workdir_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "external_workdir_grant_not_found".to_string(), - message: "External Workdir grant is unavailable".to_string(), - })?; + .db_executor() + .run(move || { + stage_api + .store + .get_external_workdir_grant(&stage_api.config.workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "external_workdir_grant_not_found".to_string(), + message: "External Workdir grant is unavailable".to_string(), + }) + }) + .await?; let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(api, grant_id) @@ -17578,9 +18117,13 @@ async fn open_current_worker_workdir_session_locked( .map_err(|error| error.error)?; } if matches!(grant.status.as_str(), "revoked" | "expired") { - cleanup_external_workdir_attachments_with_lock(api, &grant.workdir_id, Some(worker)) - .await - .map_err(|error| error.error)?; + cleanup_external_workdir_attachments_with_lock( + api, + &grant.workdir_id, + Some((worker, session_guard)), + ) + .await + .map_err(|error| error.error)?; cancel_external_workdir_expiry(api, grant_id); } if grant.status != "online" { @@ -17647,31 +18190,62 @@ async fn open_current_worker_workdir_session_locked( }); } }; + let stage_api = api.clone(); + let stage_repository_id = repository_id.to_string(); let repository = api - .require_configured_workspace_repository(repository_id) - .map_err(|error| error.error)?; + .store + .db_executor() + .run(move || { + stage_api + .require_configured_workspace_repository(&stage_repository_id) + .map_err(|error| error.error) + }) + .await?; let repository_requires_access = repository.source.kind == server_api::RepositorySourceKind::Ssh; if repository_requires_access { close_current_worker_attachment_session_locked(api, worker).await?; - let access = repository_access_request_for_workdir( - api, - runtime_id, - &workdir.workdir_id, - &format!( - "workdir-session:{}:{}:{}", - worker.runtime_id, worker.worker_id, workdir.workdir_id - ), - ) - .map_err(|error| error.error)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: runtime_id.to_string(), - code: "working_directory_remote_repository_access_required".to_string(), - message: "current Repository SSH access authority is unavailable".to_string(), - })?; - api.runtime - .authorize_working_directory_repository_access(runtime_id, access) - .map_err(RuntimeRegistryError::into_error)?; + let stage_api = api.clone(); + let stage_runtime_id = runtime_id.to_string(); + let stage_workdir_id = workdir.workdir_id.clone(); + let operation_id = format!( + "workdir-session:{}:{}:{}", + worker.runtime_id, worker.worker_id, workdir.workdir_id + ); + let stage_guard = session_guard.clone(); + let access = api + .store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + repository_access_request_for_workdir( + &stage_api, + &stage_runtime_id, + &stage_workdir_id, + &operation_id, + ) + .map_err(|error| error.error)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: stage_runtime_id, + code: "working_directory_remote_repository_access_required".to_string(), + message: "current Repository SSH access authority is unavailable".to_string(), + }) + }) + .await?; + let stage_api = api.clone(); + let stage_runtime_id = runtime_id.to_string(); + // The accepted Runtime authorization must retain the actual session lock, + // not just the async waiter that submitted it. + let stage_guard = session_guard.clone(); + host_io_executor()? + .run(move || { + let _session_guard = stage_guard; + stage_api + .runtime + .authorize_working_directory_repository_access(&stage_runtime_id, access) + .map_err(RuntimeRegistryError::into_error) + }) + .await?; } else { // The registry is an operation-session cache, not routing authority. // Close any prior alias session before opening the explicitly selected @@ -17800,10 +18374,10 @@ fn restore_logical_workdir_attachments( .map_err(ApiError::from) } -fn sync_runtime_worker_workdir_attachments( +fn prepare_runtime_worker_workdir_attachments( api: &WorkspaceApi, worker: &RuntimeWorkerRef, -) -> ApiResult<()> { +) -> ApiResult> { let attachments = api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? @@ -17846,6 +18420,14 @@ fn sync_runtime_worker_workdir_attachments( }) }) .collect::>>()?; + Ok(attachments) +} + +fn replace_runtime_worker_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + attachments: Vec, +) -> ApiResult<()> { let result = api .runtime .replace_worker_workdir_attachments(worker, attachments) @@ -17865,12 +18447,67 @@ fn sync_runtime_worker_workdir_attachments( Ok(()) } +#[cfg(test)] +fn sync_runtime_worker_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ApiResult<()> { + let attachments = prepare_runtime_worker_workdir_attachments(api, worker)?; + replace_runtime_worker_workdir_attachments(api, worker, attachments) +} + +async fn sync_runtime_worker_workdir_attachments_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + session_guard: &Arc>, +) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.clone(); + let attachments = api + .store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + prepare_runtime_worker_workdir_attachments(&stage_api, &stage_worker) + }) + .await?; + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.clone(); + host_io_executor()? + .run(move || { + let _session_guard = stage_guard; + replace_runtime_worker_workdir_attachments(&stage_api, &stage_worker, attachments) + }) + .await +} + async fn refresh_current_worker_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, + session_guard: &Arc>, ) -> Result<()> { close_current_worker_attachment_session_locked(api, worker).await?; + let stage_api = api.clone(); + let stage_worker = worker.clone(); let remaining = api + .store + .db_executor() + .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) + .await?; + if remaining.len() == 1 { + open_current_worker_workdir_session_locked(api, worker, &remaining[0], session_guard) + .await?; + } + Ok(()) +} + +fn active_filesystem_workdir_links( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Result> { + Ok(api .store .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() @@ -17884,11 +18521,7 @@ async fn refresh_current_worker_session_locked( !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) }) }) - .collect::>(); - if remaining.len() == 1 { - open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await?; - } - Ok(()) + .collect()) } #[derive(Debug, Deserialize)] @@ -17903,35 +18536,104 @@ async fn scoped_submit_backend_job_result( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; - let binding = api - .store - .worker_registry_projection(&api.config.workspace_id, &worker)? - .and_then(|projection| projection.job) - .ok_or_else(|| { - Error::InvalidInput( - "current Worker is not bound to an active Backend Job attempt".to_string(), - ) - })?; - let attempt = api - .store - .get_backend_job_attempt( + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let binding = api + .store + .worker_registry_projection(&api.config.workspace_id, &worker)? + .and_then(|projection| projection.job) + .ok_or_else(|| { + Error::InvalidInput( + "current Worker is not bound to an active Backend Job attempt".to_string(), + ) + })?; + let attempt = api + .store + .get_backend_job_attempt( + &api.config.workspace_id, + &binding.job_id, + &binding.attempt_id, + )? + .ok_or_else(|| { + Error::Store("Backend Job attempt binding disappeared".to_string()) + })?; + let acceptance = api.accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + input_digest: attempt.input_digest, + result: request.result, + }, + )?; + Ok(Json(acceptance)) + }) + .await +} + +async fn detach_worker_workdir_stage_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + workdir_id: &str, + session_guard: &Arc>, +) -> Result> { + let worker = worker.clone(); + let workdir_id = workdir_id.to_string(); + run_worker_workdir_db_stage_locked(api, session_guard, move |api| { + api.store.detach_worker_workdir( &api.config.workspace_id, - &binding.job_id, - &binding.attempt_id, - )? - .ok_or_else(|| Error::Store("Backend Job attempt binding disappeared".to_string()))?; - let acceptance = api.accept_backend_job_result( - &worker, - &BackendJobResultSubmission { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - input_digest: attempt.input_digest, - result: request.result, - }, - )?; - Ok(Json(acceptance)) + &worker, + Some(&workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) + }) + .await +} + +async fn import_current_worker_workdir_observation( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + session_guard: Option<&Arc>>, +) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let has_history = api + .store + .db_executor() + .run(move || { + stage_api + .store + .worker_workdir_link_history_exists(&stage_api.config.workspace_id, &stage_worker) + }) + .await?; + if !has_history { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_guard = session_guard.cloned(); + let observed_worker = host_io_executor()? + .run(move || { + let _session_guard = stage_guard; + stage_api + .runtime + .worker(&stage_worker) + .map_err(RuntimeRegistryError::into_error) + }) + .await?; + if !observed_worker.workdir_attachments.is_empty() { + let stage_api = api.clone(); + let stage_guard = session_guard.cloned(); + api.store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + sync_worker_observation(&stage_api, &observed_worker) + }) + .await?; + } + } + Ok(()) } async fn scoped_attach_current_worker_workdir( @@ -17940,115 +18642,139 @@ async fn scoped_attach_current_worker_workdir( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = host_io_executor()? + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &path.workspace_id)?; + Ok(current_worker_identity( + &stage_api, + &path.workspace_id, + &headers, + )?) + }) + .await?; let alias = workdir::WorkdirAttachmentAlias::new(request.alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; let workdir_id = request.working_directory_id.trim(); if workdir_id.is_empty() || workdir_id.chars().any(char::is_control) { return Err(Error::InvalidRecordId(request.working_directory_id).into()); } - let workdir = api + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_workdir_id = workdir_id.to_string(); + let capabilities = api .store - .list_workdir_registry(&api.config.workspace_id, 10_000)? - .into_iter() - .find(|workdir| workdir.workdir_id == workdir_id) - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "working_directory_not_found".to_string(), - message: format!("unknown Workdir `{workdir_id}`"), - })?; - // A failed removal must not prevent its owner from attaching a still usable - // repository checkout to resolve current blockers. Runtime binding validates - // the actual checkout; durable pending-removal reservations still fence this - // mutation. Other sources and missing/corrupted materializations stay denied. - let repository_retry = workdir.materialization_status == "pending" - && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); - if workdir.materialization_status != "present" && !repository_retry { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir `{workdir_id}` is not available for attachment ({})", - workdir.materialization_status - )) - .into()); - } - match &workdir.source { - WorkdirRegistrySource::Repository { .. } => {} - WorkdirRegistrySource::WorkspaceConfig { .. } => { - return Err(Error::WorkspacePermissionDenied( - "Use the grant-bound Workspace config attachment operation".into(), - ) - .into()); - } - WorkdirRegistrySource::ExternalGrant { grant_id } => { - let grant = api + .db_executor() + .run(move || -> ApiResult<_> { + let api = stage_api; + let worker = stage_worker; + let workdir_id = stage_workdir_id.as_str(); + let workdir = api .store - .get_external_workdir_grant(&api.config.workspace_id, grant_id)? + .list_workdir_registry(&api.config.workspace_id, 10_000)? + .into_iter() + .find(|workdir| workdir.workdir_id == workdir_id) .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir grant is unavailable".to_string(), + runtime_id: worker.runtime_id.clone(), + code: "working_directory_not_found".to_string(), + message: format!("unknown Workdir `{workdir_id}`"), })?; - let grant_expired = external_workdir_grant_is_expired(&grant)?; - let online = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connection(grant_id) - .is_some_and(|connection| { - connection.generation == grant.generation - && connection.provider_instance_id == grant.provider_instance_id - && connection.workdir_id == workdir.workdir_id - && connection - .expires_at - .is_none_or(|expires_at| expires_at > Utc::now()) - }); - if grant.status != "online" || grant_expired || !online { - return Err(Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir provider is offline or expired".to_string(), - } + // A failed removal must not prevent its owner from attaching a still usable + // repository checkout to resolve current blockers. Runtime binding validates + // the actual checkout; durable pending-removal reservations still fence this + // mutation. Other sources and missing/corrupted materializations stay denied. + let repository_retry = workdir.materialization_status == "pending" + && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); + if workdir.materialization_status != "present" && !repository_retry { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir `{workdir_id}` is not available for attachment ({})", + workdir.materialization_status + )) .into()); } - } - } - let capabilities = ticket_target_capabilities_for_workdir(&api, &worker, &workdir)?; + match &workdir.source { + WorkdirRegistrySource::Repository { .. } => {} + WorkdirRegistrySource::WorkspaceConfig { .. } => { + return Err(Error::WorkspacePermissionDenied( + "Use the grant-bound Workspace config attachment operation".into(), + ) + .into()); + } + WorkdirRegistrySource::ExternalGrant { grant_id } => { + let grant = api + .store + .get_external_workdir_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir grant is unavailable".to_string(), + })?; + let grant_expired = external_workdir_grant_is_expired(&grant)?; + let online = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connection(grant_id) + .is_some_and(|connection| { + connection.generation == grant.generation + && connection.provider_instance_id == grant.provider_instance_id + && connection.workdir_id == workdir.workdir_id + && connection + .expires_at + .is_none_or(|expires_at| expires_at > Utc::now()) + }); + if grant.status != "online" || grant_expired || !online { + return Err(Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir provider is offline or expired".to_string(), + } + .into()); + } + } + } + Ok(ticket_target_capabilities_for_workdir( + &api, &worker, &workdir, + )?) + }) + .await?; let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; - if !api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, &worker)? - { - let observed_worker = api - .runtime - .worker(&worker) - .map_err(|error| error.into_error())?; - if !observed_worker.workdir_attachments.is_empty() { - sync_worker_observation(&api, &observed_worker)?; - } - } - if let Some(existing) = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == alias.as_str()) - { - if existing.workdir_id != workdir_id { - return Err(Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} attachment alias `{alias}` is already bound to Workdir {}", - worker.runtime_id, worker.worker_id, existing.workdir_id - )) - .into()); - } - let effective_capabilities = existing.capabilities.intersection(capabilities); - if effective_capabilities != existing.capabilities { - let mut attenuated = existing.clone(); - attenuated.capabilities = effective_capabilities; - api.store.attach_worker_workdir(&attenuated)?; - } - sync_runtime_worker_workdir_attachments(&api, &worker)?; + let _session_guard = Arc::new(session_lock.lock_owned().await); + import_current_worker_workdir_observation(&api, &worker, Some(&_session_guard)).await?; + let stage_worker = worker.clone(); + let stage_alias = alias.to_string(); + let stage_workdir_id = workdir_id.to_string(); + let existing = + run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| -> ApiResult<_> { + let existing = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias); + if let Some(existing) = existing { + if existing.workdir_id != stage_workdir_id { + return Err(Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} attachment alias `{stage_alias}` is already bound to Workdir {}", + stage_worker.runtime_id, stage_worker.worker_id, existing.workdir_id + )).into()); + } + let effective_capabilities = existing.capabilities.intersection(capabilities); + if effective_capabilities != existing.capabilities { + let mut attenuated = existing; + attenuated.capabilities = effective_capabilities; + api.store.attach_worker_workdir(&attenuated)?; + } + Ok(Some(effective_capabilities)) + } else { + Ok(None) + } + }) + .await?; + if let Some(effective_capabilities) = existing { + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; api.worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from)?; return Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { workspace_id: api.config.workspace_id.clone(), @@ -18059,7 +18785,7 @@ async fn scoped_attach_current_worker_workdir( })); } close_current_worker_attachment_session_locked(&api, &worker).await?; - let link = api.store.attach_worker_workdir(&WorkerWorkdirLinkRecord { + let new_link = WorkerWorkdirLinkRecord { connection_id: String::new(), workspace_id: api.config.workspace_id.clone(), worker: worker.clone(), @@ -18068,56 +18794,46 @@ async fn scoped_attach_current_worker_workdir( capabilities, linked_at: Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), unlinked_at: None, - })?; - if let Err(error) = sync_runtime_worker_workdir_attachments(&api, &worker) { - let _ = api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ); + }; + let link = run_worker_workdir_db_stage_locked(&api, &_session_guard, move |api| { + api.store.attach_worker_workdir(&new_link) + }) + .await?; + if let Err(error) = + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await + { + let _ = + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; return Err(error); } + let stage_api = api.clone(); + let stage_worker = worker.clone(); let active_count = api .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) - .filter(|link| { - api.store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) - .ok() - .flatten() - .is_none_or(|record| { - !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) - }) + .db_executor() + .run(move || { + active_filesystem_workdir_links(&stage_api, &stage_worker).map(|links| links.len()) }) - .count(); + .await?; if active_count == 1 - && let Err(error) = open_current_worker_workdir_session_locked(&api, &worker, &link).await + && let Err(error) = + open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await { - let _ = api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - ); - let _ = sync_runtime_worker_workdir_attachments(&api, &worker); + let _ = + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; + let _ = + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await; return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from) { - api.store.detach_worker_workdir( - &api.config.workspace_id, - &worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - )?; - sync_runtime_worker_workdir_attachments(&api, &worker)?; - refresh_current_worker_session_locked(&api, &worker).await?; + detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await?; + sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; + refresh_current_worker_session_locked(&api, &worker, &_session_guard).await?; return Err(error); } Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -18247,7 +18963,10 @@ async fn scoped_detach_current_worker_workdir( validate_workspace_scope(&api, &workspace_id)?; let alias = workdir::WorkdirAttachmentAlias::new(alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; - let worker = current_worker_identity(&api, &workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = host_io_executor()? + .run(move || current_worker_identity(&stage_api, &workspace_id, &headers)) + .await?; detach_current_worker_workdir( &api, &worker, @@ -18258,6 +18977,18 @@ async fn scoped_detach_current_worker_workdir( .map(Json) } +async fn restore_worker_workdir_connection_stage_locked( + api: &WorkspaceApi, + link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, +) -> Result<()> { + let link = link.clone(); + run_worker_workdir_db_stage_locked(api, session_guard, move |api| { + api.store.restore_worker_workdir_connection(&link) + }) + .await +} + async fn detach_current_worker_workdir( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -18265,70 +18996,84 @@ async fn detach_current_worker_workdir( expected_connection_id: Option<&str>, ) -> ApiResult { let session_lock = current_worker_session_lock(api, worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_alias = alias.to_string(); + let expected_connection_id = expected_connection_id.map(str::to_string); let link = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == alias) - .ok_or_else(|| { - Error::WorkdirAttachmentConflict(format!( - "Worker {}:{} has no Workdir attachment `{alias}`", - worker.runtime_id, worker.worker_id - )) - })?; - if expected_connection_id.is_some_and(|expected| expected != link.connection_id) { - return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir attachment `{alias}` connection changed during detach" - )) - .into()); - } + .db_executor() + .run(move || -> ApiResult<_> { + let link = stage_api + .store + .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias) + .ok_or_else(|| { + Error::WorkdirAttachmentConflict(format!( + "Worker {}:{} has no Workdir attachment `{stage_alias}`", + stage_worker.runtime_id, stage_worker.worker_id + )) + })?; + if expected_connection_id + .as_ref() + .is_some_and(|expected| *expected != link.connection_id) + { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir attachment `{stage_alias}` connection changed during detach" + )) + .into()); + } + Ok(link) + }) + .await?; close_current_worker_attachment_session_locked(api, worker).await?; close_current_worker_command_sessions_for_alias_locked(api, worker, alias).await?; // Even unconditional HTTP requests condition the mutation on the lifetime just read // under the session lock, so a concurrent ledger replacement cannot be removed. - api.store.detach_worker_workdir_connection( - &api.config.workspace_id, - worker, - alias, - &link.connection_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - )?; - if let Err(error) = sync_runtime_worker_workdir_attachments(api, worker) { - api.store.restore_worker_workdir_connection(&link)?; + let stage_link = link.clone(); + run_worker_workdir_db_stage_locked(api, &_session_guard, move |api| { + api.store.detach_worker_workdir_connection( + &api.config.workspace_id, + &stage_link.worker, + &stage_link.alias, + &stage_link.connection_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) + }) + .await?; + if let Err(error) = + sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await + { + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; return Err(error); } + let stage_api = api.clone(); + let stage_worker = worker.clone(); let remaining = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) - .filter(|link| { - api.store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) - .ok() - .flatten() - .is_none_or(|record| { - !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) - }) - }) - .collect::>(); + .db_executor() + .run(move || active_filesystem_workdir_links(&stage_api, &stage_worker)) + .await?; if remaining.len() == 1 && let Err(error) = - open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await + open_current_worker_workdir_session_locked(api, worker, &remaining[0], &_session_guard) + .await { - api.store.restore_worker_workdir_connection(&link)?; - let _ = sync_runtime_worker_workdir_attachments(api, worker); + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; + let _ = sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await; return Err(error.into()); } if let Err(error) = api .worker_projection - .refresh(&worker) + .refresh_async(&worker) + .await .map_err(ApiError::from) { - api.store.restore_worker_workdir_connection(&link)?; - sync_runtime_worker_workdir_attachments(api, worker)?; - refresh_current_worker_session_locked(api, worker).await?; + restore_worker_workdir_connection_stage_locked(api, &link, &_session_guard).await?; + sync_runtime_worker_workdir_attachments_locked(api, worker, &_session_guard).await?; + refresh_current_worker_session_locked(api, worker, &_session_guard).await?; return Err(error); } Ok(server_api::CurrentWorkerWorkdirAttachmentResponse { @@ -18340,41 +19085,36 @@ async fn detach_current_worker_workdir( }) } -fn validated_current_worker_attachment( +async fn validated_current_worker_attachment_async( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, + session_guard: Option<&Arc>>, ) -> ApiResult { - let find = || -> Result> { - Ok(api + for imported in [false, true] { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let stage_alias = target_workdir.to_string(); + let link = api .store - .list_worker_workdir_links(&api.config.workspace_id, worker)? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == target_workdir)) - }; - if let Some(link) = find()? { - return Ok(link); - } - - // A Runtime can start the Worker immediately after reserving its local - // bindings, before the outer spawn handler projects them into the Backend. - // Import that exact alias set once; never substitute another attachment. - if !api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, worker)? - { - let observed_worker = api - .runtime - .worker(worker) - .map_err(|error| error.into_error())?; - if !observed_worker.workdir_attachments.is_empty() { - sync_worker_observation(api, &observed_worker)?; - if let Some(link) = find()? { - return Ok(link); - } + .db_executor() + .run(move || -> Result<_> { + Ok(stage_api + .store + .list_worker_workdir_links(&stage_api.config.workspace_id, &stage_worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.alias == stage_alias)) + }) + .await?; + if let Some(link) = link { + return Ok(link); + } + if !imported { + // A Runtime may start the Worker before the outer spawn handler + // projects its reserved aliases. Import that exact set only once. + import_current_worker_workdir_observation(api, worker, session_guard).await?; } } - Err(Error::WorkdirAttachmentConflict(format!( "Worker {}:{} has no Workdir attachment alias `{target_workdir}`", worker.runtime_id, worker.worker_id @@ -18483,19 +19223,24 @@ async fn scoped_execute_current_worker_workdir_operation( context.stage = "workspace_scope"; validate_workspace_scope(&api, &path.workspace_id)?; context.stage = "worker_identity"; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; + let stage_api = api.clone(); + let worker = host_io_executor()?.run(move || { + current_worker_identity(&stage_api, &path.workspace_id, &headers) + }).await?; context.runtime_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.runtime_id)); context.worker_id_hash = Some(crate::worker_source::diagnostic_id_hash(&worker.worker_id)); let target_workdir = request.target_workdir; let result = match request.operation { WorkdirSessionOperation::CommandStart(command) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; + let link = validated_current_worker_attachment_async( + &api, &worker, &target_workdir, Some(&_session_guard), + ).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; context.stage = "provider_dispatch"; let provider_handle = source .start_command(command) @@ -18527,7 +19272,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandStatus(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; session .command_status(provider_handle) .await @@ -18536,7 +19281,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandOutput(mut output) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &output.handle, &mut context).await?; output.handle = provider_handle; session .command_output(output) @@ -18546,7 +19291,7 @@ async fn scoped_execute_current_worker_workdir_operation( } WorkdirSessionOperation::CommandCancel(external_handle) => { let (session, provider_handle) = - current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context)?; + current_worker_command_session(&api, &worker, &target_workdir, &external_handle, &mut context).await?; session .cancel_command(provider_handle) .await @@ -18567,12 +19312,14 @@ async fn scoped_execute_current_worker_workdir_operation( | WorkdirSessionOperation::Glob(_) | WorkdirSessionOperation::Grep(_)) => { let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; + let _session_guard = Arc::new(session_lock.lock_owned().await); context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(&api, &worker, &target_workdir)?; + let link = validated_current_worker_attachment_async( + &api, &worker, &target_workdir, Some(&_session_guard), + ).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "session_open"; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await?; context.stage = "provider_dispatch"; execute_workdir_session_operation(&source, operation) .await @@ -18587,7 +19334,7 @@ async fn scoped_execute_current_worker_workdir_operation( }) } -fn current_worker_command_session( +async fn current_worker_command_session( api: &WorkspaceApi, worker: &RuntimeWorkerRef, target_workdir: &str, @@ -18595,7 +19342,7 @@ fn current_worker_command_session( context: &mut server_api::WorkdirOperationLogContext, ) -> std::result::Result<(WorkdirSessionHandle, CommandHandle), InternalWorkdirOperationError> { context.stage = "attachment_validation"; - let link = validated_current_worker_attachment(api, worker, target_workdir)?; + let link = validated_current_worker_attachment_async(api, worker, target_workdir, None).await?; context.workdir_id_hash = Some(crate::worker_source::diagnostic_id_hash(&link.workdir_id)); context.stage = "command_lookup"; let command = api @@ -19024,15 +19771,20 @@ async fn scoped_get_memory_document( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let document = api.authority.memory_document()?; - Ok(Json(MemoryDocumentResponse { - bytes: document.body_md.len(), - body_md: document.body_md, - created_at: document.created_at, - updated_at: document.updated_at, - record_source: document.record_source, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let document = api.authority.memory_document()?; + Ok(Json(MemoryDocumentResponse { + bytes: document.body_md.len(), + body_md: document.body_md, + created_at: document.created_at, + updated_at: document.updated_at, + record_source: document.record_source, + })) + }) + .await } async fn scoped_reset_legacy_memory( @@ -19055,13 +19807,18 @@ async fn scoped_reset_legacy_memory( )) .into()); } - let counts = api.store.reset_legacy_memory(&path.workspace_id)?; - Ok(Json(server_api::LegacyMemoryResetResponse { - workspace_id: path.workspace_id, - documents_deleted: counts.documents, - staging_records_deleted: counts.staging_records, - staging_resolutions_deleted: counts.staging_resolutions, - })) + api.store + .db_executor() + .run(move || { + let counts = api.store.reset_legacy_memory(&path.workspace_id)?; + Ok(Json(server_api::LegacyMemoryResetResponse { + workspace_id: path.workspace_id, + documents_deleted: counts.documents, + staging_records_deleted: counts.staging_records, + staging_resolutions_deleted: counts.staging_resolutions, + })) + }) + .await } async fn scoped_list_memory_staging( @@ -19069,13 +19826,20 @@ async fn scoped_list_memory_staging( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(list_memory_staging_from_authority( - &api.authority, - query.limit.map(usize::try_from).transpose().map_err(|_| { - Error::InvalidInput("Memory staging limit exceeds this host's usize range".to_string()) - })?, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(list_memory_staging_from_authority( + &api.authority, + query.limit.map(usize::try_from).transpose().map_err(|_| { + Error::InvalidInput( + "Memory staging limit exceeds this host's usize range".to_string(), + ) + })?, + )?)) + }) + .await } async fn scoped_memory_backend_operation( @@ -19084,21 +19848,26 @@ async fn scoped_memory_backend_operation( _context: server_api::ServerRequestContext, Json(operation): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - // The legacy memory document edit is a read/replace/write across store - // calls. Keep that resource atomic without serializing unrelated requests. - let _memory_guard = api - .memory_operation_lock - .lock() - .expect("Memory operation lock poisoned"); - let result = execute_memory_backend_operation_with_authority(&api.authority, operation); - let response = match result { - Ok(result) => MemoryBackendHttpResponse::Ok { result }, - Err(error) => MemoryBackendHttpResponse::Error { - message: sanitize_backend_error(&error.to_string()), - }, - }; - Ok(Json(response)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + // The legacy memory document edit is a read/replace/write across store + // calls. Keep that resource atomic without serializing unrelated requests. + let _memory_guard = api + .memory_operation_lock + .lock() + .expect("Memory operation lock poisoned"); + let result = execute_memory_backend_operation_with_authority(&api.authority, operation); + let response = match result { + Ok(result) => MemoryBackendHttpResponse::Ok { result }, + Err(error) => MemoryBackendHttpResponse::Error { + message: sanitize_backend_error(&error.to_string()), + }, + }; + Ok(Json(response)) + }) + .await } const SUBJEKTIV_SINGLETON_PREFIX: &str = crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX; @@ -19106,6 +19875,62 @@ const SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX: &str = crate::hosts::SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX; const SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE: &str = "subjektiv-memory-consolidation"; +// Runtime callbacks may themselves need DB authority. Synchronous Host workflows +// therefore have a separate finite lane: never monopolize the sole DB worker while +// waiting for Runtime I/O. Existing transaction helpers still finish before I/O. +// Teardown waits must not occupy the ordinary Host lane: a Job may need a +// user stop/control operation on that lane before its Run can finish. Cleanup +// therefore has its own finite owner, still independent of the DB callback lane. +fn backend_job_cleanup_executor() -> Result { + static CLEANUP: std::sync::OnceLock> = + std::sync::OnceLock::new(); + CLEANUP + .get_or_init(|| { + crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(2).unwrap(), + queued: std::num::NonZeroUsize::new(16).unwrap(), + }) + .ok() + }) + .clone() + .ok_or(Error::DbExecutorUnavailable) +} + +fn host_io_executor() -> Result { + static HOST: std::sync::OnceLock> = + std::sync::OnceLock::new(); + HOST.get_or_init(|| { + crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(2).unwrap(), + queued: std::num::NonZeroUsize::new(16).unwrap(), + }) + .ok() + }) + .clone() + .ok_or(Error::DbExecutorUnavailable) +} + +// Some existing lifecycle workflows interleave owned async resource locks with +// synchronous DB/Runtime APIs. Poll the complete workflow on the finite Host +// lane using the original runtime (never create a replacement runtime for its +// cached services). Cancellation releases neither locks nor accepted ownership. +// No SQL transaction contains an await/Runtime call, and this lane is separate +// from the DB worker needed by Runtime callbacks. Do not nest this helper. +async fn run_host_workflow( + future: impl std::future::Future> + Send + 'static, +) -> std::result::Result +where + T: Send + 'static, + E: From + Send + 'static, +{ + let runtime = + tokio::runtime::Handle::try_current().map_err(|_| E::from(Error::DbExecutorUnavailable))?; + host_io_executor() + .map_err(E::from)? + .run(move || runtime.block_on(future)) + .await +} + fn open_subjektiv_store(api: &WorkspaceApi) -> ApiResult { crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) .map_err(|error| ApiError::from(Error::Store(error.to_string()))) @@ -19323,18 +20148,23 @@ async fn scoped_create_subjektiv_subject( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { - validate_workspace_scope(&api, &path.workspace_id)?; - let role = crate::subjektiv::SubjectRole::new(request.role) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .create_subject_with_behavior(role, request.behavior_md) - .map_err(subjektiv_store_error)?; - Ok(( - StatusCode::CREATED, - Json(subjektiv_subject_response(&api, subject)?), - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let role = crate::subjektiv::SubjectRole::new(request.role) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .create_subject_with_behavior(role, request.behavior_md) + .map_err(subjektiv_store_error)?; + Ok(( + StatusCode::CREATED, + Json(subjektiv_subject_response(&api, subject)?), + )) + }) + .await } #[derive(Debug, Serialize, Deserialize)] @@ -19350,76 +20180,86 @@ async fn scoped_list_subjektiv_subjects( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let limit = query.limit.unwrap_or(20); - if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { - return Err(Error::InvalidInput(format!( - "subject list limit must be within 1..={}", - server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT - )) - .into()); - } - let cursor = query - .cursor - .as_deref() - .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) - .transpose()?; - if cursor - .as_ref() - .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) - { - return Err(Error::InvalidInput( - "subjektiv subject cursor does not match Workspace".to_string(), - ) - .into()); - } - let page = open_subjektiv_store(&api)? - .list_subjects_after( - limit, - cursor + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let limit = query.limit.unwrap_or(20); + if limit == 0 || limit > server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT { + return Err(Error::InvalidInput(format!( + "subject list limit must be within 1..={}", + server_api::SUBJEKTIV_BROWSER_MAX_LIST_LIMIT + )) + .into()); + } + let cursor = query + .cursor + .as_deref() + .map(|cursor| decode_subjektiv_cursor::("subjects", cursor)) + .transpose()?; + if cursor .as_ref() - .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), - ) - .map_err(subjektiv_store_error)?; - let next_cursor = if page.has_more { - page.items - .last() - .map(|subject| { - encode_subjektiv_cursor( - "subjects", - &SubjektivSubjectCursor { - workspace_id: path.workspace_id.clone(), - created_at: subject.created_at.clone(), - subject_id: subject.id.clone(), - }, + .is_some_and(|cursor| cursor.workspace_id != path.workspace_id) + { + return Err(Error::InvalidInput( + "subjektiv subject cursor does not match Workspace".to_string(), ) - }) - .transpose()? - } else { - None - }; - Ok(Json(server_api::SubjektivSubjectListResponse { - limit, - items: page - .items - .into_iter() - .map(|subject| subjektiv_subject_response(&api, subject)) - .collect::>>()?, - next_cursor, - has_more: page.has_more, - })) + .into()); + } + let page = open_subjektiv_store(&api)? + .list_subjects_after( + limit, + cursor + .as_ref() + .map(|cursor| (cursor.created_at.as_str(), cursor.subject_id.as_str())), + ) + .map_err(subjektiv_store_error)?; + let next_cursor = if page.has_more { + page.items + .last() + .map(|subject| { + encode_subjektiv_cursor( + "subjects", + &SubjektivSubjectCursor { + workspace_id: path.workspace_id.clone(), + created_at: subject.created_at.clone(), + subject_id: subject.id.clone(), + }, + ) + }) + .transpose()? + } else { + None + }; + Ok(Json(server_api::SubjektivSubjectListResponse { + limit, + items: page + .items + .into_iter() + .map(|subject| subjektiv_subject_response(&api, subject)) + .collect::>>()?, + next_cursor, + has_more: page.has_more, + })) + }) + .await } async fn scoped_get_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .subject(&path.subject_id) - .map_err(|error| Error::Store(error.to_string()))? - .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .subject(&path.subject_id) + .map_err(|error| Error::Store(error.to_string()))? + .ok_or_else(|| Error::SubjektivSubjectNotFound(path.subject_id.clone()))?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } async fn scoped_update_subjektiv_subject_behavior( @@ -19427,17 +20267,22 @@ async fn scoped_update_subjektiv_subject_behavior( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - crate::subjektiv::validate_subject_behavior(&request.behavior_md) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - let subject = open_subjektiv_store(&api)? - .update_subject_behavior( - &path.subject_id, - &request.expected_behavior_md, - request.behavior_md, - ) - .map_err(subjektiv_store_error)?; - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .update_subject_behavior( + &path.subject_id, + &request.expected_behavior_md, + request.behavior_md, + ) + .map_err(subjektiv_store_error)?; + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } fn require_subjektiv_subject( @@ -19458,47 +20303,53 @@ async fn scoped_get_subjektiv_resident_surface( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - let resident = store - .resident_surface(&path.subject_id) - .map_err(subjektiv_store_error)?; - let availability = match resident.availability { - crate::subjektiv::SurfaceAvailability::Ungenerated => { - server_api::SubjektivResidentSurfaceAvailability::Ungenerated - } - crate::subjektiv::SurfaceAvailability::Stale => { - server_api::SubjektivResidentSurfaceAvailability::Stale - } - crate::subjektiv::SurfaceAvailability::Failed => { - server_api::SubjektivResidentSurfaceAvailability::Failed - } - crate::subjektiv::SurfaceAvailability::Ready => { - server_api::SubjektivResidentSurfaceAvailability::Ready - } - }; - let snapshot = resident - .snapshot - .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { - snapshot_id: snapshot.id, - body_md: snapshot.body_md, - memory_refs: snapshot - .memory_refs - .into_iter() - .map(|reference| server_api::SubjektivMemoryChangeRef { - memory_id: reference.memory_id, - change_id: reference.change_id, - }) - .collect(), - built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, - created_at: snapshot.created_at, - }); - Ok(Json(server_api::SubjektivResidentSurfaceResponse { - subject_id: path.subject_id, - availability, - snapshot, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + let resident = store + .resident_surface(&path.subject_id) + .map_err(subjektiv_store_error)?; + let availability = match resident.availability { + crate::subjektiv::SurfaceAvailability::Ungenerated => { + server_api::SubjektivResidentSurfaceAvailability::Ungenerated + } + crate::subjektiv::SurfaceAvailability::Stale => { + server_api::SubjektivResidentSurfaceAvailability::Stale + } + crate::subjektiv::SurfaceAvailability::Failed => { + server_api::SubjektivResidentSurfaceAvailability::Failed + } + crate::subjektiv::SurfaceAvailability::Ready => { + server_api::SubjektivResidentSurfaceAvailability::Ready + } + }; + let snapshot = + resident + .snapshot + .map(|snapshot| server_api::SubjektivResidentSurfaceSnapshot { + snapshot_id: snapshot.id, + body_md: snapshot.body_md, + memory_refs: snapshot + .memory_refs + .into_iter() + .map(|reference| server_api::SubjektivMemoryChangeRef { + memory_id: reference.memory_id, + change_id: reference.change_id, + }) + .collect(), + built_from_memory_fingerprint: snapshot.built_from_memory_fingerprint, + created_at: snapshot.created_at, + }); + Ok(Json(server_api::SubjektivResidentSurfaceResponse { + subject_id: path.subject_id, + availability, + snapshot, + })) + }) + .await } async fn scoped_list_subjektiv_memories( @@ -19506,14 +20357,19 @@ async fn scoped_list_subjektiv_memories( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_query( - &store, - &path.subject_id, - query, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_query( + &store, + &path.subject_id, + query, + )?)) + }) + .await } async fn scoped_get_subjektiv_memory( @@ -19521,21 +20377,26 @@ async fn scoped_get_subjektiv_memory( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_read( - &store, - &path.subject_id, - server_api::SubjektivMemoryReadRequest { - memory_id: path.memory_id, - change_id: query.change_id, - offset: query.offset, - byte_offset: query.byte_offset, - limit: query.limit, - evidence_cursor: query.evidence_cursor, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_read( + &store, + &path.subject_id, + server_api::SubjektivMemoryReadRequest { + memory_id: path.memory_id, + change_id: query.change_id, + offset: query.offset, + byte_offset: query.byte_offset, + limit: query.limit, + evidence_cursor: query.evidence_cursor, + }, + )?)) + }) + .await } async fn scoped_list_subjektiv_memory_changes( @@ -19543,31 +20404,41 @@ async fn scoped_list_subjektiv_memory_changes( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let store = open_subjektiv_store(&api)?; - require_subjektiv_subject(&store, &path.subject_id)?; - Ok(Json(subjektiv_memory_list_changes( - &store, - &path.subject_id, - server_api::SubjektivMemoryListChangesRequest { - memory_id: path.memory_id, - limit: query.limit, - cursor: query.cursor, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let store = open_subjektiv_store(&api)?; + require_subjektiv_subject(&store, &path.subject_id)?; + Ok(Json(subjektiv_memory_list_changes( + &store, + &path.subject_id, + server_api::SubjektivMemoryListChangesRequest { + memory_id: path.memory_id, + limit: query.limit, + cursor: query.cursor, + }, + )?)) + }) + .await } async fn scoped_retire_subjektiv_subject( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .retire_subject(&path.subject_id) - .map_err(subjektiv_store_error)?; - // Retirement deliberately does not stop or remove the current Worker. The - // Worker lease and subject lifecycle are independent authorities. - Ok(Json(subjektiv_subject_response(&api, subject)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let subject = open_subjektiv_store(&api)? + .retire_subject(&path.subject_id) + .map_err(subjektiv_store_error)?; + // Retirement deliberately does not stop or remove the current Worker. The + // Worker lease and subject lifecycle are independent authorities. + Ok(Json(subjektiv_subject_response(&api, subject)?)) + }) + .await } async fn scoped_stage_subjektiv_candidate( @@ -19576,58 +20447,63 @@ async fn scoped_stage_subjektiv_candidate( context: server_api::ServerRequestContext, Json(mut request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )?; - let subject_id = attribution.subject_id.clone(); - let session_id = attribution.session_id.as_str(); - for source_ref in &mut request.operation.source_refs { - if source_ref - .session_id - .as_deref() - .is_some_and(|value| value != session_id) - { - return Err(Error::InvalidInput( - "candidate source_ref belongs to a different Session".to_string(), - ) - .into()); - } - source_ref.session_id = Some(session_id.to_string()); - } - if matches!( - request.operation.candidate.kind, - memory::extract::CandidateKind::Preference - ) && request.operation.evidence.iter().any(|evidence| { - !matches!( - evidence.origin.as_ref().map(|origin| &origin.kind), - Some(memory::schema::EvidenceOriginKind::HumanInput) - ) - }) { - return Err(Error::InvalidInput( - "preference candidates require exclusively HumanInput evidence".to_string(), - ) - .into()); - } - let candidate_id = Uuid::now_v7().to_string(); - let record = memory::extract::StagingRecord::from_candidate( - candidate_id, - request.operation.extract_run_id, - request.operation.source, - request.operation.candidate, - request.operation.evidence, - request.operation.source_refs, - ); - let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); - let (staged, _) = open_subjektiv_store(&api)? - .stage_candidate_with_attribution(record, attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))?; - Ok(Json(server_api::SubjektivStageCandidateResponse { - staging_id: staged.id, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )?; + let subject_id = attribution.subject_id.clone(); + let session_id = attribution.session_id.as_str(); + for source_ref in &mut request.operation.source_refs { + if source_ref + .session_id + .as_deref() + .is_some_and(|value| value != session_id) + { + return Err(Error::InvalidInput( + "candidate source_ref belongs to a different Session".to_string(), + ) + .into()); + } + source_ref.session_id = Some(session_id.to_string()); + } + if matches!( + request.operation.candidate.kind, + memory::extract::CandidateKind::Preference + ) && request.operation.evidence.iter().any(|evidence| { + !matches!( + evidence.origin.as_ref().map(|origin| &origin.kind), + Some(memory::schema::EvidenceOriginKind::HumanInput) + ) + }) { + return Err(Error::InvalidInput( + "preference candidates require exclusively HumanInput evidence".to_string(), + ) + .into()); + } + let candidate_id = Uuid::now_v7().to_string(); + let record = memory::extract::StagingRecord::from_candidate( + candidate_id, + request.operation.extract_run_id, + request.operation.source, + request.operation.candidate, + request.operation.evidence, + request.operation.source_refs, + ); + let record = crate::subjektiv::SubjectStagingRecord::attach(&subject_id, record); + let (staged, _) = open_subjektiv_store(&api)? + .stage_candidate_with_attribution(record, attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + Ok(Json(server_api::SubjektivStageCandidateResponse { + staging_id: staged.id, + })) + }) + .await } async fn scoped_record_subjektiv_session( @@ -19636,25 +20512,34 @@ async fn scoped_record_subjektiv_session( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let attribution = if request.create_if_missing { - let attribution = - subjektiv_session_attribution(&api, &path.workspace_id, &context, &request.session_id)?; - open_subjektiv_store(&api)? - .record_session_attribution(attribution) - .map_err(|error| Error::InvalidInput(error.to_string()))? - } else { - require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &request.session_id, - )? - }; - Ok(Json(server_api::SubjektivRecordSessionResponse { - subject_id: attribution.subject_id, - session_id: attribution.session_id, - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let attribution = if request.create_if_missing { + let attribution = subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )?; + open_subjektiv_store(&api)? + .record_session_attribution(attribution) + .map_err(|error| Error::InvalidInput(error.to_string()))? + } else { + require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &request.session_id, + )? + }; + Ok(Json(server_api::SubjektivRecordSessionResponse { + subject_id: attribution.subject_id, + session_id: attribution.session_id, + })) + }) + .await } const SUBJEKTIV_SESSION_DEFAULT_LIMIT: usize = 20; @@ -19748,50 +20633,61 @@ async fn scoped_subjektiv_session_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, controller, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - require_subjektiv_worker_authority( - authority, - SubjektivWorkerAuthority::Subject, - "Session history read", - )?; - let store = match open_subjektiv_store(&api) { - Ok(store) => store, - Err(error) => { - return Ok(Json(subjektiv_session_error( - server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, - error.error.to_string(), - ))); - } - }; - let response = match request.operation { - server_api::SubjektivSessionBackendOperation::List(input) => subjektiv_session_list( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - server_api::SubjektivSessionBackendOperation::Search(input) => subjektiv_session_search( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - server_api::SubjektivSessionBackendOperation::Read(input) => subjektiv_session_read( - &api, - &store, - &path.workspace_id, - &subject_id, - &controller, - input, - ), - }; - Ok(Json(response)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, controller, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + require_subjektiv_worker_authority( + authority, + SubjektivWorkerAuthority::Subject, + "Session history read", + )?; + let store = match open_subjektiv_store(&api) { + Ok(store) => store, + Err(error) => { + return Ok(Json(subjektiv_session_error( + server_api::SubjektivSessionDiagnosticCode::SubjectUnavailable, + error.error.to_string(), + ))); + } + }; + let response = match request.operation { + server_api::SubjektivSessionBackendOperation::List(input) => { + subjektiv_session_list( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + server_api::SubjektivSessionBackendOperation::Search(input) => { + subjektiv_session_search( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + server_api::SubjektivSessionBackendOperation::Read(input) => { + subjektiv_session_read( + &api, + &store, + &path.workspace_id, + &subject_id, + &controller, + input, + ) + } + }; + Ok(Json(response)) + }) + .await } fn subjektiv_session_list( @@ -20982,115 +21878,121 @@ async fn scoped_subjektiv_memory_backend( context: server_api::ServerRequestContext, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, source_worker, authority) = - subjektiv_subject_scope(&api, &path.workspace_id, &context)?; - let store = open_subjektiv_store(&api)?; - use server_api::SubjektivMemoryBackendOperation as Op; - // Authentication, live Worker/Job authority and committed Session provenance - // remain Backend-owned. Only the domain operation/projection is shared. - let (expected, label) = match &request.operation { - Op::ResidentSummary(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "resident surface read", - ), - Op::ResidentContext(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "resident subject context read", - ), - Op::ValidateProposal(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "Memory proposal validation", - ), - Op::ReceiptStatus(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "explicit candidate receipt lookup", - ), - Op::StageExplicit(_) => ( - Some(SubjektivWorkerAuthority::Subject), - "explicit candidate staging", - ), - Op::ListCandidates(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate listing", - ), - Op::ReadCandidate(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate read", - ), - Op::DecideCandidate(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "candidate decision", - ), - Op::PrepareSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface generation preparation", - ), - Op::PublishSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface publication", - ), - Op::FailSurface(_) => ( - Some(SubjektivWorkerAuthority::Consolidation), - "surface generation failure", - ), - Op::Query(_) | Op::Read(_) | Op::ListChanges(_) => (None, "recall"), - }; - if let Some(expected) = expected { - require_subjektiv_worker_authority(authority, expected, label)?; - } - let host_context = match authority { - SubjektivWorkerAuthority::Subject => { - let attribution = if let Op::StageExplicit(input) = &request.operation { - Some(require_existing_subjektiv_session_attribution( - &api, - &path.workspace_id, - &context, - &input.session_id, - )?) - } else { - None + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (subject_id, source_worker, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + let store = open_subjektiv_store(&api)?; + use server_api::SubjektivMemoryBackendOperation as Op; + // Authentication, live Worker/Job authority and committed Session provenance + // remain Backend-owned. Only the domain operation/projection is shared. + let (expected, label) = match &request.operation { + Op::ResidentSummary(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "resident surface read", + ), + Op::ResidentContext(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "resident subject context read", + ), + Op::ValidateProposal(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "Memory proposal validation", + ), + Op::ReceiptStatus(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "explicit candidate receipt lookup", + ), + Op::StageExplicit(_) => ( + Some(SubjektivWorkerAuthority::Subject), + "explicit candidate staging", + ), + Op::ListCandidates(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate listing", + ), + Op::ReadCandidate(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate read", + ), + Op::DecideCandidate(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "candidate decision", + ), + Op::PrepareSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface generation preparation", + ), + Op::PublishSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface publication", + ), + Op::FailSurface(_) => ( + Some(SubjektivWorkerAuthority::Consolidation), + "surface generation failure", + ), + Op::Query(_) | Op::Read(_) | Op::ListChanges(_) => (None, "recall"), }; - HostOperationContext::validated_body(&store, &subject_id, attribution)? - } - SubjektivWorkerAuthority::Consolidation => { - match &request.operation { - Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( - &api, - &path.workspace_id, - &source_worker, - &input.candidate_id, - )?, - _ => {} + if let Some(expected) = expected { + require_subjektiv_worker_authority(authority, expected, label)?; } - let grant = subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; - let job = api - .store - .worker_registry_projection(&path.workspace_id, &source_worker)? - .and_then(|projection| projection.job) - .map(|binding| JobAttemptBinding { - job_id: binding.job_id, - attempt_id: binding.attempt_id, - }); - HostOperationContext::validated_consolidation( + let host_context = match authority { + SubjektivWorkerAuthority::Subject => { + let attribution = if let Op::StageExplicit(input) = &request.operation { + Some(require_existing_subjektiv_session_attribution( + &api, + &path.workspace_id, + &context, + &input.session_id, + )?) + } else { + None + }; + HostOperationContext::validated_body(&store, &subject_id, attribution)? + } + SubjektivWorkerAuthority::Consolidation => { + match &request.operation { + Op::ReadCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + Op::DecideCandidate(input) => subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?, + _ => {} + } + let grant = + subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?; + let job = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|projection| projection.job) + .map(|binding| JobAttemptBinding { + job_id: binding.job_id, + attempt_id: binding.attempt_id, + }); + HostOperationContext::validated_consolidation( + &store, + &subject_id, + grant.map(|g| g.candidate_ids), + job, + )? + } + }; + Ok(Json(subjektiv::api::execute( &store, - &subject_id, - grant.map(|g| g.candidate_ids), - job, - )? - } - }; - Ok(Json(subjektiv::api::execute( - &store, - &host_context, - request.operation, - )?)) + &host_context, + request.operation, + )?)) + }) + .await } fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error { @@ -21194,10 +22096,17 @@ async fn start_subjektiv_staging_consolidation( subject_id: &str, operation: MemoryConsolidateStagingOperation, ) -> ApiResult { - let store = open_subjektiv_store(&api)?; - let (candidate_count, total_bytes) = store - .pending_staging_backlog(subject_id) - .map_err(subjektiv_store_error)?; + let job_api = api.clone(); + let subject = subject_id.to_owned(); + let (candidate_count, total_bytes) = api + .store + .db_executor() + .run(move || { + open_subjektiv_store(&job_api)? + .pending_staging_backlog(&subject) + .map_err(|error| ApiError::from(subjektiv_store_error(error))) + }) + .await?; if !subjektiv_job::retire_legacy_worker(&api, subject_id).await? { return Ok(MemoryConsolidationOutput { status: "skipped_legacy_inflight".into(), @@ -21205,10 +22114,18 @@ async fn start_subjektiv_staging_consolidation( candidate_count, total_bytes, }); } - let surface_availability = store - .resident_surface(subject_id) - .map_err(subjektiv_store_error)? - .availability; + let job_api = api.clone(); + let subject = subject_id.to_owned(); + let surface_availability = api + .store + .db_executor() + .run(move || { + open_subjektiv_store(&job_api)? + .resident_surface(&subject) + .map(|surface| surface.availability) + .map_err(|error| ApiError::from(subjektiv_store_error(error))) + }) + .await?; if !subject_consolidation_required( candidate_count, total_bytes, @@ -21235,7 +22152,10 @@ async fn start_subjektiv_staging_consolidation( }); } - subjektiv_job::dispatch(&api, subject_id, candidate_count, total_bytes) + let subject = subject_id.to_owned(); + host_io_executor()? + .run(move || subjektiv_job::dispatch(&api, &subject, candidate_count, total_bytes)) + .await } async fn scoped_memory_consolidation( @@ -21243,8 +22163,12 @@ async fn scoped_memory_consolidation( AxumPath(path): AxumPath, Json(operation): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(start_memory_staging_consolidation(api, operation)?)) + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(start_memory_staging_consolidation(api, operation)?)) + }) + .await } fn start_memory_staging_consolidation( @@ -21444,72 +22368,92 @@ async fn scoped_list_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::catalog(&state).map(Json).map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::catalog(&state).map(Json).map_err(skill_api_error) + }) + .await } async fn scoped_lint_skills( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::lint(&state).map(Json).map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::lint(&state).map(Json).map_err(skill_api_error) + }) + .await } async fn scoped_get_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::detail(&state, &path.name) - .map(Json) - .map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::detail(&state, &path.name) + .map(Json) + .map_err(skill_api_error) + }) + .await } async fn scoped_activate_skill( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let state = api - .config_store - .load_workspace_config(&path.workspace_id)? - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Workspace {} has no active config snapshot", - path.workspace_id - )) - })?; - skills::activation(&state, &path.name) - .map(Json) - .map_err(skill_api_error) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let state = api + .config_store + .load_workspace_config(&path.workspace_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workspace {} has no active config snapshot", + path.workspace_id + )) + })?; + skills::activation(&state, &path.name) + .map(Json) + .map_err(skill_api_error) + }) + .await } fn skill_api_error(error: skills::SkillError) -> ApiError { @@ -21559,8 +22503,13 @@ async fn scoped_query_objectives( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.query_objectives(query)?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.query_objectives(query)?)) + }) + .await } async fn scoped_show_objective( @@ -21568,10 +22517,15 @@ async fn scoped_show_objective( AxumPath(path): AxumPath, Json(query): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority.show_objective(&path.objective_id, query)?, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority.show_objective(&path.objective_id, query)?, + )) + }) + .await } async fn scoped_create_objective( @@ -21579,15 +22533,20 @@ async fn scoped_create_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.create_objective( - ObjectiveCreateInput { - title: request.title, - body_md: request.body_md, - state: request.state, - linked_tickets: request.linked_tickets, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.create_objective( + ObjectiveCreateInput { + title: request.title, + body_md: request.body_md, + state: request.state, + linked_tickets: request.linked_tickets, + }, + )?)) + }) + .await } async fn scoped_edit_objective( @@ -21595,16 +22554,21 @@ async fn scoped_edit_objective( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.edit_objective( - &path.objective_id, - ObjectiveEditInput { - title: request.title, - old_string: request.old_string, - new_string: request.new_string, - replace_all: request.replace_all, - }, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.edit_objective( + &path.objective_id, + ObjectiveEditInput { + title: request.title, + old_string: request.old_string, + new_string: request.new_string, + replace_all: request.replace_all, + }, + )?)) + }) + .await } async fn scoped_set_objective_state( @@ -21612,11 +22576,16 @@ async fn scoped_set_objective_state( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json( - api.authority - .set_objective_state(&path.objective_id, &request.state)?, - )) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + api.authority + .set_objective_state(&path.objective_id, &request.state)?, + )) + }) + .await } async fn scoped_link_objective_ticket( @@ -21624,22 +22593,32 @@ async fn scoped_link_objective_ticket( AxumPath(path): AxumPath, Json(request): Json, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.link_objective_ticket( - &path.objective_id, - &request.ticket_id, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.link_objective_ticket( + &path.objective_id, + &request.ticket_id, + )?)) + }) + .await } async fn scoped_unlink_objective_ticket( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(api.authority.unlink_objective_ticket( - &path.objective_id, - &path.ticket_id, - )?)) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(api.authority.unlink_objective_ticket( + &path.objective_id, + &path.ticket_id, + )?)) + }) + .await } async fn scoped_list_repositories( @@ -22147,6 +23126,9 @@ async fn worker_remove_contract( .map_err(|error| { let status = match error { crate::worker_source::WorkerMutationSourceProofError::Replay => StatusCode::CONFLICT, + crate::worker_source::WorkerMutationSourceProofError::Unavailable => { + StatusCode::SERVICE_UNAVAILABLE + } crate::worker_source::WorkerMutationSourceProofError::Authority(_) => { StatusCode::INTERNAL_SERVER_ERROR } @@ -22161,7 +23143,8 @@ async fn worker_remove_contract( ) })?; let worker = - WorkspaceWorker::resolve(api, &request.target_runtime_id, &request.target_worker_id) + WorkspaceWorker::resolve_async(api, &request.target_runtime_id, &request.target_worker_id) + .await .map_err(|error| match error { Error::UnknownWorker { .. } => server_api::WorkerRemoveApiError::new( StatusCode::NOT_FOUND.as_u16(), @@ -22217,22 +23200,27 @@ async fn scoped_get_workspace_worker( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker_ref = api - .store - .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? - .ok_or_else(|| Error::UnknownWorker { - worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), - })?; - let workers = workers_response(api.clone())?; - workers - .items - .into_iter() - .find(|worker| { - worker.runtime_id == worker_ref.runtime_id && worker.worker_id == worker_ref.worker_id + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let worker_ref = api + .store + .resolve_worker_resource_reference(&api.config.workspace_id, &path.worker_ref)? + .ok_or_else(|| Error::UnknownWorker { + worker: RuntimeWorkerRef::new("unknown", &path.worker_ref), + })?; + let workers = workers_response(api.clone())?; + workers + .items + .into_iter() + .find(|worker| { + worker.runtime_id == worker_ref.runtime_id + && worker.worker_id == worker_ref.worker_id + }) + .map(Json) + .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) }) - .map(Json) - .ok_or_else(|| Error::UnknownWorker { worker: worker_ref }.into()) + .await } async fn discover_workspace_workers( @@ -22355,8 +23343,13 @@ async fn scoped_workspace_orchestrator_status( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - Ok(Json(workspace_orchestrator_response(&api, "observed"))) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json(workspace_orchestrator_response(&api, "observed"))) + }) + .await } async fn list_known_workers( @@ -22364,36 +23357,42 @@ async fn list_known_workers( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let limit = api.config.max_records.clamp(1, 500); - let grants = - api.store - .list_active_worker_control_grants(&path.workspace_id, &controller, limit + 1)?; - let truncated = grants.len() > limit; - let mut items = Vec::with_capacity(grants.len().min(limit)); - for grant in grants.into_iter().take(limit) { - let summary = api - .runtime - .worker(&grant.subject) - .map_err(|error| error.into_error())?; - items.push(server_api::WorkerControlRecord { - subject: server_api::WorkerControlSubject { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - relation: grant.relation, - origin: grant.origin, - permissions: grant.permissions, - summary: worker_launch_worker_summary(summary), - }); - } - Ok(Json(server_api::WorkerControlListResponse { - workspace_id: path.workspace_id, - items, - truncated, - })) + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let limit = api.config.max_records.clamp(1, 500); + let grants = api.store.list_active_worker_control_grants( + &path.workspace_id, + &controller, + limit + 1, + )?; + let truncated = grants.len() > limit; + let mut items = Vec::with_capacity(grants.len().min(limit)); + for grant in grants.into_iter().take(limit) { + let summary = api + .runtime + .worker(&grant.subject) + .map_err(|error| error.into_error())?; + items.push(server_api::WorkerControlRecord { + subject: server_api::WorkerControlSubject { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + relation: grant.relation, + origin: grant.origin, + permissions: grant.permissions, + summary: worker_launch_worker_summary(summary), + }); + } + Ok(Json(server_api::WorkerControlListResponse { + workspace_id: path.workspace_id, + items, + truncated, + })) + }) + .await } fn scoped_worker_control_operation_id(controller: &RuntimeWorkerRef, operation_id: &str) -> String { @@ -22528,7 +23527,9 @@ async fn send_known_worker_input( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; let request = worker_input_request_from_api(request)?; let idle_only = request.kind == WorkerInputKind::User; let result = worker.input(&context, request).await?; @@ -22564,7 +23565,9 @@ async fn cancel_known_worker( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; Ok(Json(worker_lifecycle_result_to_api( worker .cancel(&context, worker_lifecycle_request_from_api(request)) @@ -22583,7 +23586,9 @@ async fn stop_known_worker( let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let worker = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; Ok(Json(worker_lifecycle_result_to_api( worker .stop(&context, worker_lifecycle_request_from_api(request)) @@ -22617,41 +23622,45 @@ async fn scoped_list_worker_observation_sessions( AxumPath(path): AxumPath, headers: HeaderMap, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let sessions = api - .store - .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? - .into_iter() - .filter(|grant| { - grant - .permissions - .iter() - .any(|permission| permission == "observe") + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; + let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + let sessions = api + .store + .list_active_worker_control_grants(&path.workspace_id, &controller, 100)? + .into_iter() + .filter(|grant| { + grant + .permissions + .iter() + .any(|permission| permission == "observe") + }) + .filter_map(|grant| { + let worker = api.runtime.worker(&grant.subject).ok()?; + if matches!( + worker.state.as_str(), + "stopped" | "failed" | "rejected" | "disconnected" + ) { + return None; + } + Some(server_api::WorkerObservationSubject { + subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { + runtime_id: grant.subject.runtime_id, + worker_id: grant.subject.worker_id, + }, + display_name: worker.display_name, + relation: grant.relation, + status: worker.state, + }) + }) + .collect::>(); + Ok(Json(server_api::WorkerObservationSessionsResponse { + sessions, + })) }) - .filter_map(|grant| { - let worker = api.runtime.worker(&grant.subject).ok()?; - if matches!( - worker.state.as_str(), - "stopped" | "failed" | "rejected" | "disconnected" - ) { - return None; - } - Some(server_api::WorkerObservationSubject { - subject: server_api::WorkerObservationSubjectRef::RuntimeWorker { - runtime_id: grant.subject.runtime_id, - worker_id: grant.subject.worker_id, - }, - display_name: worker.display_name, - relation: grant.relation, - status: worker.state, - }) - }) - .collect::>(); - Ok(Json(server_api::WorkerObservationSessionsResponse { - sessions, - })) + .await } async fn scoped_capture_worker_observation_session( @@ -22739,6 +23748,8 @@ async fn scoped_start_workspace_orchestrator( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { + host_io_executor()?.run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; let _guard = api.orchestrator_spawn_lock.lock().map_err(|_| { ApiError::with_diagnostics( @@ -22867,6 +23878,8 @@ async fn scoped_start_workspace_orchestrator( drop(_guard); dispatch_orchestrator_queue_attention(&api, &worker.worker, false); Ok(Json(workspace_orchestrator_response(&api, disposition))) + + }).await } fn worker_launch_worker_summary(worker: InternalWorkerSummary) -> WorkerLaunchWorkerSummary { @@ -22998,13 +24011,17 @@ async fn scoped_list_runtime_working_directories( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) + host_io_executor()? + .run(move || { + let (items, diagnostics) = runtime_working_directory_summaries(&api, &path.runtime_id)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) + }) + .await } async fn scoped_create_runtime_working_directory( @@ -23028,7 +24045,11 @@ async fn scoped_runtime_working_directory_detail( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) + host_io_executor()? + .run(move || { + working_directory_detail_for_runtime(api, &path.runtime_id, &path.working_directory_id) + }) + .await } fn workdir_removal_source_actor( @@ -23114,19 +24135,24 @@ async fn cleanup_runtime_working_directory( ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; require_active_workspace_runtime_binding(&api, &path.runtime_id).await?; - let registered_runtime = registered_workdir_runtime_id(&api, &path.working_directory_id)?; - if registered_runtime != path.runtime_id { - return Err(ApiError::from(Error::WorkspacePermissionDenied( - "Workdir does not belong to the requested Runtime".to_string(), - ))); - } - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) + host_io_executor()? + .run(move || { + let registered_runtime = + registered_workdir_runtime_id(&api, &path.working_directory_id)?; + if registered_runtime != path.runtime_id { + return Err(ApiError::from(Error::WorkspacePermissionDenied( + "Workdir does not belong to the requested Runtime".to_string(), + ))); + } + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) + }) + .await } fn external_workdir_expires_at( @@ -23178,9 +24204,18 @@ fn schedule_external_workdir_expiry( } Err(error) => { tracing::error!(grant_id = %task_grant_id, error = ?error, "Backend External Workdir expiry cleanup requires retry"); + let stage_api = task_api.clone(); + let stage_grant_id = task_grant_id.clone(); let retryable = task_api .store - .get_external_workdir_grant(&task_api.config.workspace_id, &task_grant_id) + .db_executor() + .run(move || { + stage_api.store.get_external_workdir_grant( + &stage_api.config.workspace_id, + &stage_grant_id, + ) + }) + .await .map_or(true, |grant| grant.is_some()); if !retryable { break; @@ -23245,6 +24280,18 @@ async fn stop_external_workdir_provider( async fn fence_external_workdir_grant_expired( api: &WorkspaceApi, grant_id: &str, +) -> ApiResult { + let stage_api = api.clone(); + let grant_id = grant_id.to_string(); + api.store + .db_executor() + .run(move || fence_external_workdir_grant_expired_stage(&stage_api, &grant_id)) + .await +} + +fn fence_external_workdir_grant_expired_stage( + api: &WorkspaceApi, + grant_id: &str, ) -> ApiResult { loop { let mut grant = api @@ -23314,6 +24361,8 @@ async fn scoped_create_external_workdir_grant( Extension(actor): Extension, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { + api.store.db_executor().run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; let display_name = request.display_name.trim(); if display_name.is_empty() @@ -23383,6 +24432,8 @@ async fn scoped_create_external_workdir_grant( StatusCode::CREATED, Json(external_workdir_grant_response(&grant)?), )) + + }).await } fn external_workdir_grant_response( @@ -23407,16 +24458,21 @@ async fn scoped_external_workdir_provider_ws( AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ws: WebSocketUpgrade, ) -> ApiResult { - validate_workspace_scope(&api, &workspace_id)?; - let mut grant = api - .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir provider connection must use the grant creator's authenticated client".to_string(), - ).into()); - } + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); + let mut grant = api.store.db_executor().run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &stage_workspace_id)?; + let grant = stage_api.store + .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir provider connection must use the grant creator's authenticated client".to_string(), + ).into()); + } + Ok(grant) + }).await?; let expired = external_workdir_grant_is_expired(&grant)?; if expired { grant = fence_external_workdir_grant_expired(&api, &grant_id).await?; @@ -23563,81 +24619,102 @@ async fn serve_external_workdir_provider( .await; return; } - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !api + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let registration_generation = registration.generation; + let registration_capabilities = registration.capabilities; + let activation = api .store - .activate_external_workdir_grant( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration.generation, - &now, - ) - .unwrap_or(false) - { - let _ = socket.close().await; - return; - } - if let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_connected", - &initial_grant.grant_id, - None, - "online", - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, registration.generation - ), - &now, - ) { - let _ = api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &initial_grant.provider_instance_id, - registration.generation, - "offline", - &now, - ); - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = registration.generation, - %error, - "failed to audit External Workdir provider activation" - ); - let _ = socket.close().await; - return; - } - let expires_at = match external_workdir_expires_at(&initial_grant) { - Ok(value) => value, - Err(_) => { + .db_executor() + .run(move || -> Result<_> { + let api = stage_api; + let initial_grant = stage_grant; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !api + .store + .activate_external_workdir_grant( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration_generation, + &now, + ) + .unwrap_or(false) + { + return Ok(None); + } + if let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_connected", + &initial_grant.grant_id, + None, + "online", + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, registration_generation + ), + &now, + ) { + let _ = api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &initial_grant.provider_instance_id, + registration_generation, + "offline", + &now, + ); + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = registration_generation, + %error, + "failed to audit External Workdir provider activation" + ); + return Ok(None); + } + let expires_at = match external_workdir_expires_at(&initial_grant) { + Ok(value) => value, + Err(_) => return Ok(None), + }; + let (sender, commands) = tokio::sync::mpsc::channel(32); + let connection = Arc::new(ExternalProviderConnection { + grant_id: initial_grant.grant_id.clone(), + workdir_id: initial_grant.workdir_id.clone(), + provider_instance_id: initial_grant.provider_instance_id.clone(), + generation: registration_generation, + expires_at, + capabilities: registration_capabilities, + admission: Arc::new(tokio::sync::Semaphore::new(16)), + shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + sender, + }); + // Publish with the accepted activation stage, never in its cancellable waiter. + if api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connected(connection.clone()) + .is_some() + { + return Ok(None); + } + Ok(Some((connection, commands))) + }) + .await; + let (connection, mut commands) = match activation { + Ok(Some(activated)) => activated, + Ok(None) => { + let _ = socket.close().await; + return; + } + Err(error) => { + tracing::error!(grant_id = %initial_grant.grant_id, %error, + "External Workdir provider activation stage failed"); let _ = socket.close().await; return; } }; - let (sender, mut commands) = tokio::sync::mpsc::channel(32); - let connection = Arc::new(ExternalProviderConnection { - grant_id: initial_grant.grant_id.clone(), - workdir_id: initial_grant.workdir_id.clone(), - provider_instance_id: initial_grant.provider_instance_id.clone(), - generation: registration.generation, - expires_at, - capabilities: registration.capabilities, - admission: Arc::new(tokio::sync::Semaphore::new(16)), - shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), - sender, - }); - if api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connected(connection.clone()) - .is_some() - { - let _ = socket.close().await; - return; - } let registered = ExternalWorkdirServerFrame::current(ExternalWorkdirServerMessage::Registered { generation: connection.generation, @@ -23645,19 +24722,36 @@ async fn serve_external_workdir_provider( max_in_flight_operations: 16, }); if !send_external_workdir_server_frame(&mut socket, ®istered).await { - let _ = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - if let Err(error) = api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &connection.provider_instance_id, - connection.generation, - "offline", - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let stage_connection = connection.clone(); + if let Err(error) = api + .store + .db_executor() + .run(move || { + stage_api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&stage_connection.grant_id, stage_connection.generation); + stage_api.store.update_external_workdir_grant_state( + &stage_grant.workspace_id, + &stage_grant.grant_id, + &stage_connection.provider_instance_id, + stage_connection.generation, + "offline", + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await + { + // Rejected admission must not leave a dead socket discoverable. + // The generation-bound in-memory fence is safe even for an unknown + // DB outcome; it neither guesses nor retries the durable mutation. + api.external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); tracing::error!( grant_id = %initial_grant.grant_id, generation = connection.generation, @@ -23807,55 +24901,81 @@ async fn serve_external_workdir_provider( } }; pending.clear(); - let _ = api - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .disconnect(&connection.grant_id, connection.generation); - let state_updated = match api.store.update_external_workdir_grant_state( - &initial_grant.workspace_id, - &initial_grant.grant_id, - &connection.provider_instance_id, - connection.generation, - terminal_status, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) { + let stage_api = api.clone(); + let stage_grant = initial_grant.clone(); + let stage_connection = connection.clone(); + let terminal_update = api + .store + .db_executor() + .run(move || -> Result<_> { + let api = stage_api; + let initial_grant = stage_grant; + let connection = stage_connection; + let _ = api + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + let state_updated = match api.store.update_external_workdir_grant_state( + &initial_grant.workspace_id, + &initial_grant.grant_id, + &connection.provider_instance_id, + connection.generation, + terminal_status, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) { + Ok(updated) => updated, + Err(error) => { + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to persist External Workdir provider terminal state" + ); + false + } + }; + if state_updated + && let Err(error) = api.store.record_external_workdir_audit( + &initial_grant.workspace_id, + "account", + &initial_grant.created_by, + "external_workdir_provider_disconnected", + &initial_grant.grant_id, + None, + terminal_status, + &format!( + "workdir_id={} generation={}", + initial_grant.workdir_id, connection.generation + ), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + { + tracing::error!( + grant_id = %initial_grant.grant_id, + generation = connection.generation, + status = terminal_status, + %error, + "failed to audit External Workdir provider terminal state" + ); + } + Ok(state_updated) + }) + .await; + let state_updated = match terminal_update { Ok(updated) => updated, Err(error) => { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to persist External Workdir provider terminal state" - ); + api.external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .disconnect(&connection.grant_id, connection.generation); + tracing::error!(grant_id = %initial_grant.grant_id, + generation = connection.generation, status = terminal_status, %error, + "External Workdir provider terminal stage failed"); false } }; - if state_updated - && let Err(error) = api.store.record_external_workdir_audit( - &initial_grant.workspace_id, - "account", - &initial_grant.created_by, - "external_workdir_provider_disconnected", - &initial_grant.grant_id, - None, - terminal_status, - &format!( - "workdir_id={} generation={}", - initial_grant.workdir_id, connection.generation - ), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - { - tracing::error!( - grant_id = %initial_grant.grant_id, - generation = connection.generation, - status = terminal_status, - %error, - "failed to audit External Workdir provider terminal state" - ); - } if state_updated && matches!(terminal_status, "expired" | "revoked") && let Err(error) = @@ -23875,17 +24995,27 @@ async fn scoped_get_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); let mut grant = api .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir grant inspection requires the grant creator".to_string(), - ) - .into()); - } + .db_executor() + .run(move || -> ApiResult<_> { + validate_workspace_scope(&stage_api, &stage_workspace_id)?; + let grant = stage_api + .store + .get_external_workdir_grant(&stage_workspace_id, &stage_grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir grant inspection requires the grant creator".to_string(), + ) + .into()); + } + Ok(grant) + }) + .await?; let expired = !matches!(grant.status.as_str(), "revoked" | "expired") && external_workdir_grant_is_expired(&grant)?; if expired { @@ -23912,44 +25042,57 @@ async fn scoped_revoke_external_workdir_grant( Extension(actor): Extension, AxumPath((workspace_id, grant_id)): AxumPath<(String, String)>, ) -> ApiResult> { - validate_workspace_scope(&api, &workspace_id)?; - let mut grant = api + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.clone(); + let stage_grant_id = grant_id.clone(); + let grant = api .store - .get_external_workdir_grant(&workspace_id, &grant_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; - if grant.created_by != actor.account_id { - return Err(Error::WorkspacePermissionDenied( - "External Workdir revoke requires the grant creator".to_string(), - ) - .into()); - } - let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - if !matches!(grant.status.as_str(), "revoked" | "expired") { - api.store.update_external_workdir_grant_state( - &workspace_id, - &grant_id, - &grant.provider_instance_id, - grant.generation, - "revoked", - &updated_at, - )?; - grant.status = "revoked".to_string(); - grant.updated_at = updated_at; - api.store.record_external_workdir_audit( - &workspace_id, - "account", - &actor.account_id, - "external_workdir_grant_revoked", - &grant_id, - None, - "revoked", - &format!( - "workdir_id={} generation={}", - grant.workdir_id, grant.generation - ), - &grant.updated_at, - )?; - } + .db_executor() + .run(move || -> ApiResult<_> { + let api = stage_api; + let workspace_id = stage_workspace_id; + let grant_id = stage_grant_id; + validate_workspace_scope(&api, &workspace_id)?; + let mut grant = api + .store + .get_external_workdir_grant(&workspace_id, &grant_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir grant".to_string()))?; + if grant.created_by != actor.account_id { + return Err(Error::WorkspacePermissionDenied( + "External Workdir revoke requires the grant creator".to_string(), + ) + .into()); + } + let updated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + if !matches!(grant.status.as_str(), "revoked" | "expired") { + api.store.update_external_workdir_grant_state( + &workspace_id, + &grant_id, + &grant.provider_instance_id, + grant.generation, + "revoked", + &updated_at, + )?; + grant.status = "revoked".to_string(); + grant.updated_at = updated_at; + api.store.record_external_workdir_audit( + &workspace_id, + "account", + &actor.account_id, + "external_workdir_grant_revoked", + &grant_id, + None, + "revoked", + &format!( + "workdir_id={} generation={}", + grant.workdir_id, grant.generation + ), + &grant.updated_at, + )?; + } + Ok(grant) + }) + .await?; // Cleanup is deliberately retried for terminal grants. A prior revoke may // have committed its fence before a Runtime/session projection failure. let cleanup = cleanup_external_workdir_attachments(&api, &grant.workdir_id).await; @@ -23974,70 +25117,88 @@ async fn cleanup_external_workdir_attachments( async fn cleanup_external_workdir_attachments_with_lock( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<&RuntimeWorkerRef>, + already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, ) -> ApiResult<()> { - let workdir = api + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); + let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); + let grant_id = api .store - .get_workdir_registry(&api.config.workspace_id, workdir_id)? - .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; - let grant_id = workdir.source.external_grant_id().ok_or_else(|| { - Error::InvalidInput("Workdir is not backed by an External grant".to_string()) - })?; - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "pending", - None, - &now, - )?; - - match release_external_workdir_attachments(api, workdir_id, already_locked_worker).await { - Ok(()) => { - let completed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "completed", + .db_executor() + .run(move || -> Result<_> { + let _session_guard = stage_guard; + let workdir = stage_api + .store + .get_workdir_registry(&stage_api.config.workspace_id, &stage_workdir_id)? + .ok_or_else(|| Error::InvalidInput("Unknown External Workdir".to_string()))?; + let grant_id = workdir + .source + .external_grant_id() + .ok_or_else(|| { + Error::InvalidInput("Workdir is not backed by an External grant".to_string()) + })? + .to_string(); + stage_api.store.update_external_workdir_cleanup_state( + &stage_api.config.workspace_id, + &grant_id, + "pending", None, - &completed_at, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), )?; - api.store.record_external_workdir_audit( - &api.config.workspace_id, + Ok(grant_id) + }) + .await?; + + let release = + release_external_workdir_attachments(api, workdir_id, already_locked_worker).await; + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); + let succeeded = release.is_ok(); + let stage_guard = already_locked_worker.map(|(_, guard)| guard.clone()); + api.store + .db_executor() + .run(move || -> Result<()> { + let _session_guard = stage_guard; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let (state, category, event, outcome, detail) = if succeeded { + ( + "completed", + None, + "external_workdir_cleanup_completed", + "completed", + format!("workdir_id={stage_workdir_id}"), + ) + } else { + ( + "retry_required", + Some("attachment_release_failed"), + "external_workdir_cleanup_attention_required", + "attention_required", + format!("workdir_id={stage_workdir_id} category=attachment_release_failed"), + ) + }; + stage_api.store.update_external_workdir_cleanup_state( + &stage_api.config.workspace_id, + &grant_id, + state, + category, + &now, + )?; + stage_api.store.record_external_workdir_audit( + &stage_api.config.workspace_id, "system", "workspace-backend", - "external_workdir_cleanup_completed", - grant_id, + event, + &grant_id, None, - "completed", - &format!("workdir_id={workdir_id}"), - &completed_at, + outcome, + &detail, + &now, )?; Ok(()) - } - Err(error) => { - let failed_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - api.store.update_external_workdir_cleanup_state( - &api.config.workspace_id, - grant_id, - "retry_required", - Some("attachment_release_failed"), - &failed_at, - )?; - api.store.record_external_workdir_audit( - &api.config.workspace_id, - "system", - "workspace-backend", - "external_workdir_cleanup_attention_required", - grant_id, - None, - "attention_required", - &format!("workdir_id={workdir_id} category=attachment_release_failed"), - &failed_at, - )?; - Err(error) - } - } + }) + .await?; + release } #[cfg(test)] @@ -24098,17 +25259,28 @@ async fn release_external_workdir_attachment_locked( api: &WorkspaceApi, workdir_id: &str, link: &WorkerWorkdirLinkRecord, + session_guard: &Arc>, ) -> ApiResult<()> { if link.unlinked_at.is_none() { close_current_worker_attachment_session_locked(api, &link.worker).await?; close_current_worker_command_sessions_for_alias_locked(api, &link.worker, &link.alias) .await?; - api.store.detach_worker_workdir( - &api.config.workspace_id, - &link.worker, - Some(workdir_id), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; + let stage_api = api.clone(); + let stage_worker = link.worker.clone(); + let stage_workdir_id = workdir_id.to_string(); + let stage_guard = session_guard.clone(); + api.store + .db_executor() + .run(move || { + let _session_guard = stage_guard; + stage_api.store.detach_worker_workdir( + &stage_api.config.workspace_id, + &stage_worker, + Some(&stage_workdir_id), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await?; #[cfg(test)] if take_external_workdir_cleanup_failure(workdir_id) { return Err(Error::RuntimeOperationFailed { @@ -24123,9 +25295,10 @@ async fn release_external_workdir_attachment_locked( // have committed detach before Runtime persistence or projection failed. #[cfg(test)] record_external_workdir_cleanup_reconciliation(workdir_id); - sync_runtime_worker_workdir_attachments(api, &link.worker)?; + sync_runtime_worker_workdir_attachments_locked(api, &link.worker, session_guard).await?; api.worker_projection - .refresh(&link.worker) + .refresh_async(&link.worker) + .await .map_err(ApiError::from)?; Ok(()) } @@ -24133,23 +25306,34 @@ async fn release_external_workdir_attachment_locked( async fn release_external_workdir_attachments( api: &WorkspaceApi, workdir_id: &str, - already_locked_worker: Option<&RuntimeWorkerRef>, + already_locked_worker: Option<(&RuntimeWorkerRef, &Arc>)>, ) -> ApiResult<()> { + let stage_api = api.clone(); + let stage_workdir_id = workdir_id.to_string(); let Some(link) = api .store - .latest_workdir_worker_link(&api.config.workspace_id, workdir_id)? + .db_executor() + .run(move || { + stage_api + .store + .latest_workdir_worker_link(&stage_api.config.workspace_id, &stage_workdir_id) + }) + .await? else { return Ok(()); }; // The newest link is the only possible active attachment and remains the // durable reconciliation target if detach committed before Runtime update. // Missing/ambiguous Runtime state remains a retryable cleanup failure. - if already_locked_worker == Some(&link.worker) { - release_external_workdir_attachment_locked(api, workdir_id, &link).await?; + if let Some((worker, guard)) = + already_locked_worker.filter(|(worker, _)| *worker == &link.worker) + { + debug_assert_eq!(worker, &link.worker); + release_external_workdir_attachment_locked(api, workdir_id, &link, guard).await?; } else { let lock = current_worker_session_lock(api, &link.worker); - let _guard = lock.lock().await; - release_external_workdir_attachment_locked(api, workdir_id, &link).await?; + let guard = Arc::new(lock.lock_owned().await); + release_external_workdir_attachment_locked(api, workdir_id, &link, &guard).await?; } Ok(()) } @@ -24158,14 +25342,19 @@ async fn scoped_list_working_directories( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let (items, diagnostics) = working_directory_summaries(&api)?; - let items = items.into_iter().map(Into::into).collect(); - Ok(Json(BrowserWorkingDirectoryListResponse { - workspace_id: api.config.workspace_id.clone(), - items, - diagnostics: working_directory_diagnostics(diagnostics), - })) + api.store + .db_executor() + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let (items, diagnostics) = working_directory_summaries(&api)?; + let items = items.into_iter().map(Into::into).collect(); + Ok(Json(BrowserWorkingDirectoryListResponse { + workspace_id: api.config.workspace_id.clone(), + items, + diagnostics: working_directory_diagnostics(diagnostics), + })) + }) + .await } async fn scoped_create_working_directory( @@ -24180,40 +25369,46 @@ async fn scoped_working_directory_detail( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let record = api - .store - .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "working_directory_not_found".to_string(), - message: format!("Unknown Workdir `{}`", path.working_directory_id), - })?; - if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { - let grant = api - .store - .get_workspace_config_grant(&api.config.workspace_id, grant_id)? - .ok_or_else(|| { - Error::WorkspacePermissionDenied("Workspace config grant unavailable".into()) - })?; - return Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: workspace_config::summary(&grant, &record), - diagnostics: vec![], - })); - } - if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { - working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) - } else { - let item = projected_workdir_summary_from_record(&api, &record)?; - Ok(Json(BrowserWorkingDirectoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: None, - item: item.into(), - diagnostics: Vec::new(), - })) - } + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let record = api + .store + .get_workdir_registry(&api.config.workspace_id, &path.working_directory_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "working_directory_not_found".to_string(), + message: format!("Unknown Workdir `{}`", path.working_directory_id), + })?; + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Workspace config grant unavailable".into(), + ) + })?; + return Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: workspace_config::summary(&grant, &record), + diagnostics: vec![], + })); + } + if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { + working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) + } else { + let item = projected_workdir_summary_from_record(&api, &record)?; + Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: item.into(), + diagnostics: Vec::new(), + })) + } + }) + .await } async fn cleanup_working_directory( @@ -24223,13 +25418,17 @@ async fn cleanup_working_directory( request: WorkingDirectoryRemovalRequest, ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; - execute_workdir_removal( - &api, - &path.working_directory_id, - &source_actor, - &request.reason, - ) - .map_err(ApiError::from) + host_io_executor()? + .run(move || { + execute_workdir_removal( + &api, + &path.working_directory_id, + &source_actor, + &request.reason, + ) + .map_err(ApiError::from) + }) + .await } fn registered_workdir_runtime_id( @@ -24301,12 +25500,49 @@ fn finish_rejected_workdir_create_operation( Ok(failure_code) } -async fn create_workspace_working_directory( +async fn fail_workdir_create_stage( + api: &WorkspaceApi, + reserved: &WorkdirCreateOperationRecord, + request_fingerprint: &str, + failure: &str, + remove_registry: bool, +) -> Result { + let stage_api = api.clone(); + let reserved = reserved.clone(); + let request_fingerprint = request_fingerprint.to_string(); + let failure = failure.to_string(); + api.store + .db_executor() + .run(move || { + if remove_registry { + let _ = stage_api.store.delete_workdir_registry( + &reserved.workspace_id, + &reserved.working_directory_id, + ); + } + stage_api.config_store.finish_workdir_create_operation( + &reserved.workspace_id, + &reserved.operation_id, + &request_fingerprint, + false, + Some(&failure), + &now_registry_timestamp(), + ) + }) + .await +} + +fn prepare_workspace_workdir_create( api: &WorkspaceApi, workspace_id: &str, route_runtime_id: Option<&str>, mut request: BrowserWorkingDirectoryCreateRequest, -) -> ApiResult<(StatusCode, Json)> { +) -> ApiResult<( + WorkingDirectoryRequest, + String, + String, + WorkdirCreateOperationRecord, +)> { validate_workspace_scope(api, workspace_id)?; request.display_name = request .display_name @@ -24465,55 +25701,107 @@ async fn create_workspace_working_directory( })? }; - if reserved.state == "succeeded" { - return working_directory_detail_for_runtime( - api.clone(), - &reserved.resolved_runtime_id, - &reserved.working_directory_id, - ) - .map(|Json(response)| { - ( - StatusCode::OK, - Json(BrowserWorkingDirectoryCreateResponse { - workspace_id: response.workspace_id, - runtime_id: response - .runtime_id - .expect("Repository Workdir detail has Runtime"), - item: response.item, - diagnostics: response.diagnostics, - http_status: StatusCode::OK.as_u16(), - }), + Ok(( + working_directory_request, + operation_id, + request_fingerprint, + reserved, + )) +} + +async fn create_workspace_working_directory( + api: &WorkspaceApi, + workspace_id: &str, + route_runtime_id: Option<&str>, + request: BrowserWorkingDirectoryCreateRequest, +) -> ApiResult<(StatusCode, Json)> { + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.to_string(); + let route_runtime_id = route_runtime_id.map(str::to_string); + let (mut working_directory_request, operation_id, request_fingerprint, reserved) = api + .store + .db_executor() + .run(move || { + prepare_workspace_workdir_create( + &stage_api, + &stage_workspace_id, + route_runtime_id.as_deref(), + request, ) - }); + }) + .await?; + + if reserved.state == "succeeded" { + let stage_api = api.clone(); + return host_io_executor()? + .run(move || { + working_directory_detail_for_runtime( + stage_api, + &reserved.resolved_runtime_id, + &reserved.working_directory_id, + ) + .map(|Json(response)| { + ( + StatusCode::OK, + Json(BrowserWorkingDirectoryCreateResponse { + workspace_id: response.workspace_id, + runtime_id: response + .runtime_id + .expect("Repository Workdir detail has Runtime"), + item: response.item, + diagnostics: response.diagnostics, + http_status: StatusCode::OK.as_u16(), + }), + ) + }) + }) + .await; } let reserved = if reserved.state == "failed" { - api.config_store.begin_failed_workdir_create_retry( - workspace_id, - &operation_id, - &request_fingerprint, - &now_registry_timestamp(), - )? + let stage_api = api.clone(); + let stage_workspace_id = workspace_id.to_string(); + let stage_operation_id = operation_id.clone(); + let stage_fingerprint = request_fingerprint.clone(); + api.store + .db_executor() + .run(move || { + stage_api.config_store.begin_failed_workdir_create_retry( + &stage_workspace_id, + &stage_operation_id, + &stage_fingerprint, + &now_registry_timestamp(), + ) + }) + .await? } else { reserved }; - let runtime = match api - .runtime - .list_runtimes(usize::MAX) - .items - .into_iter() - .find(|runtime| runtime.runtime_id == reserved.resolved_runtime_id) + let stage_api = api.clone(); + let stage_runtime_id = reserved.resolved_runtime_id.clone(); + let runtime = match host_io_executor()? + .run(move || { + Ok::<_, Error>( + stage_api + .runtime + .list_runtimes(usize::MAX) + .items + .into_iter() + .find(|runtime| runtime.runtime_id == stage_runtime_id), + ) + }) + .await? { Some(runtime) => runtime, None => { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "runtime_unavailable", false, - Some("runtime_unavailable"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(ApiError::with_diagnostics( Error::UnknownRuntime(reserved.resolved_runtime_id), vec![RuntimeDiagnostic { @@ -24525,14 +25813,14 @@ async fn create_workspace_working_directory( } }; if runtime.kind == "remote_http" && runtime.status != "connected" { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "runtime_unavailable", false, - Some("runtime_unavailable"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: reserved.resolved_runtime_id, @@ -24547,38 +25835,50 @@ async fn create_workspace_working_directory( )); } - if let Err(error) = authorize_repository_materialization_operation( - api, - &reserved, - &request_fingerprint, - &mut working_directory_request, - ) { - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let stage_fingerprint = request_fingerprint.clone(); + let (authorization, mut working_directory_request) = api + .store + .db_executor() + .run(move || { + let authorization = authorize_repository_materialization_operation( + &stage_api, + &stage_reserved, + &stage_fingerprint, + &mut working_directory_request, + ); + Ok::<_, Error>((authorization, working_directory_request)) + }) + .await?; + if let Err(error) = authorization { + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, + "working_directory_remote_repository_access_required", false, - Some("working_directory_remote_repository_access_required"), - &now_registry_timestamp(), - )?; + ) + .await?; return Err(error); } working_directory_request.backend_workdir_id = Some(reserved.working_directory_id.clone()); - let existing = match api.runtime.working_directory( - &reserved.resolved_runtime_id, - &reserved.working_directory_id, - ) { + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let existing = match host_io_executor()? + .run(move || { + Ok::<_, Error>(stage_api.runtime.working_directory( + &stage_reserved.resolved_runtime_id, + &stage_reserved.working_directory_id, + )) + }) + .await? + { Ok(existing) => existing, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, - &request_fingerprint, - false, - Some(&failure_code), - &now_registry_timestamp(), - )?; + fail_workdir_create_stage(api, &reserved, &request_fingerprint, &failure_code, false) + .await?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -24592,24 +25892,29 @@ async fn create_workspace_working_directory( let result = if existing.working_directory.is_some() { existing } else { - match api - .runtime - .create_working_directory(&reserved.resolved_runtime_id, working_directory_request) + let stage_api = api.clone(); + let stage_runtime_id = reserved.resolved_runtime_id.clone(); + match host_io_executor()? + .run(move || { + Ok::<_, Error>( + stage_api + .runtime + .create_working_directory(&stage_runtime_id, working_directory_request), + ) + }) + .await? { Ok(result) => result, Err(error) => { let failure_code = workdir_runtime_failure_code(&error); - let _ = api - .store - .delete_workdir_registry(workspace_id, &reserved.working_directory_id); - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, + fail_workdir_create_stage( + api, + &reserved, &request_fingerprint, - false, - Some(&failure_code), - &now_registry_timestamp(), - )?; + &failure_code, + true, + ) + .await?; return Err(ApiError::with_diagnostics( error.into_error(), vec![RuntimeDiagnostic { @@ -24622,18 +25927,30 @@ async fn create_workspace_working_directory( } }; let Some(working_directory) = result.working_directory else { - let _ = api - .store - .delete_workdir_registry(workspace_id, &reserved.working_directory_id); let mut diagnostics = result.diagnostics; - let failure_code = finish_rejected_workdir_create_operation( - &api.config_store, - workspace_id, - &operation_id, - &request_fingerprint, - &diagnostics, - &now_registry_timestamp(), - )?; + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let stage_fingerprint = request_fingerprint.clone(); + let (failure_code, returned_diagnostics) = api + .store + .db_executor() + .run(move || -> Result<_> { + let _ = stage_api.store.delete_workdir_registry( + &stage_reserved.workspace_id, + &stage_reserved.working_directory_id, + ); + let failure_code = finish_rejected_workdir_create_operation( + &stage_api.config_store, + &stage_reserved.workspace_id, + &stage_reserved.operation_id, + &stage_fingerprint, + &diagnostics, + &now_registry_timestamp(), + )?; + Ok((failure_code, diagnostics)) + }) + .await?; + diagnostics = returned_diagnostics; if !diagnostics.iter().any(|diagnostic| { diagnostic.severity == HostDiagnosticSeverity::Error && diagnostic.code == failure_code }) { @@ -24655,21 +25972,29 @@ async fn create_workspace_working_directory( diagnostics, )); }; - let record = workdir_record_from_summary( - api, - &reserved.resolved_runtime_id, - &working_directory.summary, - ); - api.store.upsert_workdir_registry(&record)?; - api.config_store.finish_workdir_create_operation( - workspace_id, - &operation_id, - &request_fingerprint, - true, - None, - &now_registry_timestamp(), - )?; - let summary = projected_workdir_summary_from_record(api, &record)?; + let stage_api = api.clone(); + let stage_reserved = reserved.clone(); + let summary = api + .store + .db_executor() + .run(move || -> ApiResult<_> { + let record = workdir_record_from_summary( + &stage_api, + &stage_reserved.resolved_runtime_id, + &working_directory.summary, + ); + stage_api.store.upsert_workdir_registry(&record)?; + stage_api.config_store.finish_workdir_create_operation( + &stage_reserved.workspace_id, + &stage_reserved.operation_id, + &request_fingerprint, + true, + None, + &now_registry_timestamp(), + )?; + Ok(projected_workdir_summary_from_record(&stage_api, &record)?) + }) + .await?; Ok(( StatusCode::CREATED, Json(BrowserWorkingDirectoryCreateResponse { @@ -25249,7 +26574,7 @@ async fn set_worker_retention( runtime_worker_id: String, pinned: bool, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &runtime_worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &runtime_worker_id).await?; Ok(Json( worker .set_pinned(&WorkerOperationContext::Backend, pinned) @@ -25513,7 +26838,11 @@ async fn execute_runtime_cleanup_with_context( } context.authorize_workspace(api).await?; - let plan = build_runtime_cleanup_plan(api, runtime_id)?; + let plan_api = api.clone(); + let plan_runtime = runtime_id.to_owned(); + let plan = host_io_executor()? + .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) + .await?; // Rebuild the actual candidate set immediately before executing. The digest // binds targets, dependencies and blockers; per-target authority is checked again below. if request.expected_plan_digest != plan.digest { @@ -25536,20 +26865,36 @@ async fn execute_runtime_cleanup_with_context( candidate.runtime_id.clone(), candidate.runtime_worker_id.clone(), ); - if let Some(claim) = api + let authority_api = api.clone(); + let target = worker.clone(); + let (claim, assignment) = api .store - .get_pending_ticket_assignment_operation_for_worker(&api.config.workspace_id, &worker)? - { + .db_executor() + .run(move || -> Result<_> { + Ok(( + authority_api + .store + .get_pending_ticket_assignment_operation_for_worker( + &authority_api.config.workspace_id, + &target, + )?, + authority_api + .store + .get_active_ticket_role_assignment_for_worker( + &authority_api.config.workspace_id, + &target, + )?, + )) + }) + .await?; + if let Some(claim) = claim { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_claim_pending", &format!("Worker has pending Ticket claim {}", claim.operation_id), )); } - if let Some(assignment) = api - .store - .get_active_ticket_role_assignment_for_worker(&api.config.workspace_id, &worker)? - { + if let Some(assignment) = assignment { return Err(cleanup_api_error( runtime_id, "workspace_cleanup_worker_assigned", @@ -25575,7 +26920,8 @@ async fn execute_runtime_cleanup_with_context( )); } parse_runtime_worker_id_for_registry(&candidate.runtime_worker_id)?; - WorkspaceWorker::resolve(api, &worker.runtime_id, &worker.worker_id)? + WorkspaceWorker::resolve_async(api, &worker.runtime_id, &worker.worker_id) + .await? .delete_from_plan(context, candidate) .await?; api.workdir_session_locks @@ -25614,12 +26960,19 @@ async fn execute_runtime_cleanup_with_context( }); } CleanupTargetKind::WorkdirCleanCleanup | CleanupTargetKind::WorkdirRecordDelete => { - let removal = execute_workdir_removal( - api, - candidate.workdir_id.as_str(), - &format!("runtime-cleanup:{runtime_id}"), - &format!("cleanup target {}", candidate.target_id), - )?; + let removal_api = api.clone(); + let removal_candidate = candidate.clone(); + let requester = format!("runtime-cleanup:{runtime_id}"); + let removal = host_io_executor()? + .run(move || { + execute_workdir_removal( + &removal_api, + removal_candidate.workdir_id.as_str(), + &requester, + &format!("cleanup target {}", removal_candidate.target_id), + ) + }) + .await?; let (status, message) = match removal.disposition { WorkingDirectoryRemovalDisposition::Removed => ( "deleted", @@ -25665,7 +27018,11 @@ async fn execute_runtime_cleanup_with_context( )); } - let plan_after = build_runtime_cleanup_plan(api, runtime_id)?; + let plan_api = api.clone(); + let plan_runtime = runtime_id.to_owned(); + let plan_after = host_io_executor()? + .run(move || build_runtime_cleanup_plan(&plan_api, &plan_runtime)) + .await?; let executed_at = now_registry_timestamp(); Ok(RuntimeCleanupExecutionResponse { workspace_id: api.config.workspace_id.clone(), @@ -25937,7 +27294,9 @@ async fn scoped_restore_runtime_worker_with_context( request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let resolved = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let resolved = + WorkspaceWorker::resolve_async(&api, &path.worker.runtime_id, &path.worker.worker_id) + .await?; let _authorization = resolved.authorize_operation(&context, "restore").await?; let runtime_id = resolved.identity().runtime_id.clone(); let worker_id = resolved.identity().worker_id.clone(); @@ -25970,9 +27329,13 @@ async fn scoped_runtime_cleanup_plan( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; - Ok(Json(plan)) + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let plan = build_runtime_cleanup_plan(&api, path.runtime_id.as_str())?; + Ok(Json(plan)) + }) + .await } #[cfg(test)] @@ -26017,56 +27380,64 @@ async fn scoped_create_attachment_upload_grant( AxumPath(path): AxumPath, Query(query): Query, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - resolve_workspace_worker_reference(&api, &path.worker.runtime_id, &path.worker.worker_id)?; - let now = attachment_upload_now_ms()?; - let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); - let upload_id = match query.upload_id { - Some(upload_id) => Uuid::parse_str(&upload_id) - .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? - .to_string(), - None => Uuid::now_v7().to_string(), - }; - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - grants.retain(|_, grant| grant.expires_at_ms >= now); - if let Some(existing) = grants.get(&upload_id) { - if existing.workspace_id == path.workspace_id - && existing.runtime_id == path.worker.runtime_id - && existing.worker_id == path.worker.worker_id - && existing.account_id == actor.account_id - && existing.file_name == query.file_name - && existing.media_type == query.media_type - { - return Ok(Json(AttachmentUploadGrantResponse { + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + resolve_workspace_worker_reference( + &api, + &path.worker.runtime_id, + &path.worker.worker_id, + )?; + let now = attachment_upload_now_ms()?; + let expires_at_ms = now.saturating_add(ATTACHMENT_UPLOAD_GRANT_TTL_MS); + let upload_id = match query.upload_id { + Some(upload_id) => Uuid::parse_str(&upload_id) + .map_err(|_| Error::InvalidInput("attachment upload id must be a UUID".into()))? + .to_string(), + None => Uuid::now_v7().to_string(), + }; + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + grants.retain(|_, grant| grant.expires_at_ms >= now); + if let Some(existing) = grants.get(&upload_id) { + if existing.workspace_id == path.workspace_id + && existing.runtime_id == path.worker.runtime_id + && existing.worker_id == path.worker.worker_id + && existing.account_id == actor.account_id + && existing.file_name == query.file_name + && existing.media_type == query.media_type + { + return Ok(Json(AttachmentUploadGrantResponse { + upload_id, + expires_at_ms: existing.expires_at_ms, + })); + } + return Err(Error::RepositoryConflict( + "attachment upload id was reused with different intent".into(), + ) + .into()); + } + grants.insert( + upload_id.clone(), + AttachmentUploadGrant { + workspace_id: path.workspace_id, + runtime_id: path.worker.runtime_id, + worker_id: path.worker.worker_id, + account_id: actor.account_id, + file_name: query.file_name, + media_type: query.media_type, + expires_at_ms, + state: AttachmentUploadGrantState::Pending, + }, + ); + Ok(Json(AttachmentUploadGrantResponse { upload_id, - expires_at_ms: existing.expires_at_ms, - })); - } - return Err(Error::RepositoryConflict( - "attachment upload id was reused with different intent".into(), - ) - .into()); - } - grants.insert( - upload_id.clone(), - AttachmentUploadGrant { - workspace_id: path.workspace_id, - runtime_id: path.worker.runtime_id, - worker_id: path.worker.worker_id, - account_id: actor.account_id, - file_name: query.file_name, - media_type: query.media_type, - expires_at_ms, - state: AttachmentUploadGrantState::Pending, - }, - ); - Ok(Json(AttachmentUploadGrantResponse { - upload_id, - expires_at_ms, - })) + expires_at_ms, + })) + }) + .await } async fn upload_runtime_worker_file_contract( @@ -26151,49 +27522,54 @@ async fn scoped_cancel_attachment_upload( Extension(actor): Extension, AxumPath(path): AxumPath, ) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let completed = { - let mut grants = api - .attachment_upload_grants - .lock() - .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; - let grant = grants - .get_mut(&path.upload_id) - .ok_or_else(|| Error::InvalidInput("attachment upload grant is unknown".into()))?; - if grant.workspace_id != path.workspace_id - || grant.runtime_id != path.runtime_id - || grant.worker_id != path.worker_id - || grant.account_id != actor.account_id - { - return Err(Error::WorkspacePermissionDenied( - "attachment upload grant scope mismatch".into(), - ) - .into()); - } - match &grant.state { - AttachmentUploadGrantState::Pending => { - grants.remove(&path.upload_id); - None + host_io_executor()? + .run(move || { + validate_workspace_scope(&api, &path.workspace_id)?; + let completed = { + let mut grants = api + .attachment_upload_grants + .lock() + .map_err(|_| Error::Store("attachment upload grant lock is poisoned".into()))?; + let grant = grants.get_mut(&path.upload_id).ok_or_else(|| { + Error::InvalidInput("attachment upload grant is unknown".into()) + })?; + if grant.workspace_id != path.workspace_id + || grant.runtime_id != path.runtime_id + || grant.worker_id != path.worker_id + || grant.account_id != actor.account_id + { + return Err(Error::WorkspacePermissionDenied( + "attachment upload grant scope mismatch".into(), + ) + .into()); + } + match &grant.state { + AttachmentUploadGrantState::Pending => { + grants.remove(&path.upload_id); + None + } + AttachmentUploadGrantState::Uploading => { + grant.state = AttachmentUploadGrantState::Cancelled; + None + } + AttachmentUploadGrantState::Cancelled => None, + AttachmentUploadGrantState::Completed(file) => { + let file = file.clone(); + grants.remove(&path.upload_id); + Some(file) + } + } + }; + if let Some(file) = completed { + let worker = + resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &file.artifact_id) + .map_err(|error| error.into_error())?; } - AttachmentUploadGrantState::Uploading => { - grant.state = AttachmentUploadGrantState::Cancelled; - None - } - AttachmentUploadGrantState::Cancelled => None, - AttachmentUploadGrantState::Completed(file) => { - let file = file.clone(); - grants.remove(&path.upload_id); - Some(file) - } - } - }; - if let Some(file) = completed { - let worker = resolve_workspace_worker_reference(&api, &path.runtime_id, &path.worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &file.artifact_id) - .map_err(|error| error.into_error())?; - } - Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) + Ok(Json(AttachmentUploadCancelResponse { cancelled: true })) + }) + .await } async fn scoped_delete_runtime_worker_uploaded_file( @@ -26270,8 +27646,13 @@ async fn post_auth_bootstrap_user( State(api): State, Json(request): Json, ) -> ApiResult> { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - Ok(Json(AuthUserResponse { user })) + api.store + .db_executor() + .run(move || { + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + Ok(Json(AuthUserResponse { user })) + }) + .await } async fn post_passkey_registration_options( @@ -26279,48 +27660,53 @@ async fn post_passkey_registration_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let exclude_credentials = passkeys_for_user(&api, &user.user_id)? - .into_iter() - .map(|passkey| passkey.cred_id().clone()) - .collect(); - let user_unique_id = Uuid::now_v7(); - let (public_key, state) = webauthn - .start_passkey_registration( - user_unique_id, - &user.handle, - &user.display_name, - Some(exclude_credentials), - ) - .map_err(|error| auth_error("webauthn_registration_options_failed", &error.to_string()))?; - let challenge_id = new_id("webauthn-registration"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_registration".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some( - serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?, - ), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyRegistrationOptionsResponse { - challenge_id, - public_key, - })) + api.store + .db_executor() + .run(move || { + let user = ensure_user_account(&api, &request.handle, request.display_name.as_deref())?; + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let exclude_credentials = passkeys_for_user(&api, &user.user_id)? + .into_iter() + .map(|passkey| passkey.cred_id().clone()) + .collect(); + let user_unique_id = Uuid::now_v7(); + let (public_key, state) = webauthn + .start_passkey_registration( + user_unique_id, + &user.handle, + &user.display_name, + Some(exclude_credentials), + ) + .map_err(|error| { + auth_error("webauthn_registration_options_failed", &error.to_string()) + })?; + let challenge_id = new_id("webauthn-registration"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_registration".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some(serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyRegistrationOptionsResponse { + challenge_id, + public_key, + })) + }) + .await } fn complete_passkey_registration( @@ -26397,50 +27783,53 @@ async fn post_passkey_login_options( headers: HeaderMap, Json(request): Json, ) -> ApiResult> { - let user = match request.handle.as_deref() { - Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, - None => api.store.any_user()?, - } - .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; - let passkeys = passkeys_for_user(&api, &user.user_id)?; - if passkeys.is_empty() { - return Err(auth_error( - "passkey_not_registered", - "user has no registered passkey credentials", - ) - .into()); - } - let header_origin = request_origin(&headers); - let requested_origin = request - .browser_origin - .as_deref() - .or(header_origin.as_deref()); - let auth = auth_config_for_origin(&api.config, requested_origin)?; - let webauthn = webauthn_for_auth(&auth)?; - let (public_key, state) = webauthn - .start_passkey_authentication(&passkeys) - .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; - let challenge_id = new_id("webauthn-login"); - api.store.put_auth_challenge(&AuthChallengeRecord { - challenge_id: challenge_id.clone(), - ceremony: "passkey_login".to_string(), - challenge: challenge_id.clone(), - user_id: Some(user.user_id), - rp_id: auth.rp_id, - origin: auth.origin, - state_json: Some( - serde_json::to_string(&state).map_err(|error| { - auth_error("webauthn_state_serialize_failed", &error.to_string()) - })?, - ), - expires_at: rfc3339_after(Duration::minutes(5)), - created_at: crate::auth::now_rfc3339(), - consumed_at: None, - })?; - Ok(Json(PasskeyLoginOptionsResponse { - challenge_id, - public_key, - })) + api.store + .db_executor() + .run(move || { + let user = match request.handle.as_deref() { + Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?, + None => api.store.any_user()?, + } + .ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?; + let passkeys = passkeys_for_user(&api, &user.user_id)?; + if passkeys.is_empty() { + return Err(auth_error( + "passkey_not_registered", + "user has no registered passkey credentials", + ) + .into()); + } + let header_origin = request_origin(&headers); + let requested_origin = request + .browser_origin + .as_deref() + .or(header_origin.as_deref()); + let auth = auth_config_for_origin(&api.config, requested_origin)?; + let webauthn = webauthn_for_auth(&auth)?; + let (public_key, state) = webauthn + .start_passkey_authentication(&passkeys) + .map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?; + let challenge_id = new_id("webauthn-login"); + api.store.put_auth_challenge(&AuthChallengeRecord { + challenge_id: challenge_id.clone(), + ceremony: "passkey_login".to_string(), + challenge: challenge_id.clone(), + user_id: Some(user.user_id), + rp_id: auth.rp_id, + origin: auth.origin, + state_json: Some(serde_json::to_string(&state).map_err(|error| { + auth_error("webauthn_state_serialize_failed", &error.to_string()) + })?), + expires_at: rfc3339_after(Duration::minutes(5)), + created_at: crate::auth::now_rfc3339(), + consumed_at: None, + })?; + Ok(Json(PasskeyLoginOptionsResponse { + challenge_id, + public_key, + })) + }) + .await } fn complete_passkey_login( @@ -26558,48 +27947,53 @@ async fn post_device_login_start( State(api): State, Json(request): Json, ) -> ApiResult> { - const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; + api.store + .db_executor() + .run(move || { + const MAX_CODE_ALLOCATION_ATTEMPTS: usize = 8; - let auth = auth_public_config(&api.config); - let verification_uri = format!( - "{}/login/device", - auth.public_base_url.trim_end_matches('/') - ); - for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { - let device_code = mint_secret("yoi_device"); - let user_code = new_user_code(); - let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); - let flow = DeviceLoginFlowRecord { - device_code: device_code.clone(), - user_code: user_code.clone(), - verification_uri: verification_uri.clone(), - client_name: request.client_name.clone(), - user_id: None, - api_token_id: None, - issued_access_token: None, - created_at: crate::auth::now_rfc3339(), - expires_at: rfc3339_after(Duration::minutes(10)), - approved_at: None, - consumed_at: None, - }; - if !api.store.try_create_device_login_flow(&flow)? { - continue; - } - return Ok(Json(DeviceLoginStartResponse { - device_code, - user_code, - verification_uri, - verification_uri_complete, - expires_in: 600, - interval: 5, - })); - } + let auth = auth_public_config(&api.config); + let verification_uri = format!( + "{}/login/device", + auth.public_base_url.trim_end_matches('/') + ); + for _ in 0..MAX_CODE_ALLOCATION_ATTEMPTS { + let device_code = mint_secret("yoi_device"); + let user_code = new_user_code(); + let verification_uri_complete = format!("{verification_uri}?user_code={user_code}"); + let flow = DeviceLoginFlowRecord { + device_code: device_code.clone(), + user_code: user_code.clone(), + verification_uri: verification_uri.clone(), + client_name: request.client_name.clone(), + user_id: None, + api_token_id: None, + issued_access_token: None, + created_at: crate::auth::now_rfc3339(), + expires_at: rfc3339_after(Duration::minutes(10)), + approved_at: None, + consumed_at: None, + }; + if !api.store.try_create_device_login_flow(&flow)? { + continue; + } + return Ok(Json(DeviceLoginStartResponse { + device_code, + user_code, + verification_uri, + verification_uri_complete, + expires_in: 600, + interval: 5, + })); + } - Err(auth_error( - "device_login_code_allocation_failed", - "could not allocate a unique device login user code", - ) - .into()) + Err(auth_error( + "device_login_code_allocation_failed", + "could not allocate a unique device login user code", + ) + .into()) + }) + .await } fn approve_device_login( @@ -26665,41 +28059,48 @@ async fn post_device_login_poll( State(api): State, Json(request): Json, ) -> ApiResult> { - let Some(flow) = api - .store - .get_device_login_flow_by_device_code(&request.device_code)? - else { - return Err(auth_error("unknown_device_login", "device login flow does not exist").into()); - }; - if is_expired(&flow.expires_at) { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Expired, - access_token: None, - token_type: None, - })); - } - if flow.approved_at.is_none() { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Pending, - access_token: None, - token_type: None, - })); - } - let Some(consumed) = api - .store - .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? - else { - return Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Consumed, - access_token: None, - token_type: None, - })); - }; - Ok(Json(DeviceLoginPollResponse { - status: DeviceLoginPollStatus::Approved, - access_token: consumed.issued_access_token, - token_type: Some(DeviceAccessTokenType::Bearer), - })) + api.store + .db_executor() + .run(move || { + let Some(flow) = api + .store + .get_device_login_flow_by_device_code(&request.device_code)? + else { + return Err( + auth_error("unknown_device_login", "device login flow does not exist").into(), + ); + }; + if is_expired(&flow.expires_at) { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Expired, + access_token: None, + token_type: None, + })); + } + if flow.approved_at.is_none() { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Pending, + access_token: None, + token_type: None, + })); + } + let Some(consumed) = api + .store + .consume_device_login_token(&request.device_code, &crate::auth::now_rfc3339())? + else { + return Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Consumed, + access_token: None, + token_type: None, + })); + }; + Ok(Json(DeviceLoginPollResponse { + status: DeviceLoginPollStatus::Approved, + access_token: consumed.issued_access_token, + token_type: Some(DeviceAccessTokenType::Bearer), + })) + }) + .await } fn logout_browser_session( @@ -26893,7 +28294,7 @@ async fn resolve_actor( headers: &HeaderMap, ) -> ApiResult> { let cookie_name = auth_public_config(&api.config).cookie_name; - Ok(resolve_request_actor(api.store.as_ref(), headers, &cookie_name).await?) + Ok(resolve_request_actor(api.store.clone(), headers, &cookie_name).await?) } async fn require_actor(api: &ServerAuthApi, headers: &HeaderMap) -> ApiResult { @@ -26985,103 +28386,130 @@ async fn list_tickets( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(30).clamp(1, 100); - let states = query - .states - .as_deref() - .map(|states| { - states - .split(',') - .filter(|state| !state.is_empty()) - .map(str::to_string) - .collect::>() + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(30).clamp(1, 100); + let states = query + .states + .as_deref() + .map(|states| { + states + .split(',') + .filter(|state| !state.is_empty()) + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default(); + let crate::records::TicketSummaryPage { + items, + page, + invalid_records, + record_authority, + } = api + .authority + .list_ticket_page(crate::records::TicketListProjectionRequest { + states, + limit: Some(limit), + cursor: query.cursor, + })?; + Ok(Json(crate::records::TicketListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + page, + invalid_records, + record_authority, + })) }) - .unwrap_or_default(); - let crate::records::TicketSummaryPage { - items, - page, - invalid_records, - record_authority, - } = api - .authority - .list_ticket_page(crate::records::TicketListProjectionRequest { - states, - limit: Some(limit), - cursor: query.cursor, - })?; - Ok(Json(crate::records::TicketListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - page, - invalid_records, - record_authority, - })) + .await } async fn get_ticket( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - Ok(Json(api.authority.ticket(&id)?)) + api.store + .db_executor() + .run(move || Ok(Json(api.authority.ticket(&id)?))) + .await } async fn list_objectives( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(api.config.max_records).min(1000); - let ProjectRecordList { - items, - invalid_records, - record_authority, - } = api.authority.list_objectives(limit)?; - Ok(Json(server_api::ObjectiveListResponse { - workspace_id: api.config.workspace_id, - limit, - items, - invalid_records, - record_authority, - })) + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(api.config.max_records).min(1000); + let ProjectRecordList { + items, + invalid_records, + record_authority, + } = api.authority.list_objectives(limit)?; + Ok(Json(server_api::ObjectiveListResponse { + workspace_id: api.config.workspace_id, + limit, + items, + invalid_records, + record_authority, + })) + }) + .await } async fn get_objective( State(api): State, AxumPath(id): AxumPath, ) -> ApiResult> { - Ok(Json(api.authority.objective(&id)?)) + api.store + .db_executor() + .run(move || Ok(Json(api.authority.objective(&id)?))) + .await } async fn list_repositories( State(api): State, ) -> ApiResult> { - let RepositoryListProjection { items, diagnostics } = api.live_repository_reader()?.list(); - Ok(Json(RepositoryListResponse { - workspace_id: api.config.workspace_id, - items, - source: "workspace-control-plane".to_string(), - diagnostics, - })) + api.store + .db_executor() + .run(move || { + let RepositoryListProjection { items, diagnostics } = + api.live_repository_reader()?.list(); + Ok(Json(RepositoryListResponse { + workspace_id: api.config.workspace_id, + items, + source: "workspace-control-plane".to_string(), + diagnostics, + })) + }) + .await } async fn repository_detail( State(api): State, AxumPath(repository_key): AxumPath, ) -> ApiResult> { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let item = repository_lookup( - api.live_repository_reader()? - .summary_by_key(&repository_key), - )?; - Ok(Json(RepositoryDetailResponse { - workspace_id: api.config.workspace_id.clone(), - item, - source: "workspace-control-plane".to_string(), - })) + api.store + .db_executor() + .run(move || { + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let item = repository_lookup( + api.live_repository_reader()? + .summary_by_key(&repository_key), + )?; + Ok(Json(RepositoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + item, + source: "workspace-control-plane".to_string(), + })) + }) + .await } async fn repository_log( @@ -27089,63 +28517,79 @@ async fn repository_log( AxumPath(repository_key): AxumPath, Query(query): Query, ) -> ApiResult> { - server_api::validate_repository_key(&repository_key).map_err(|error| { - ApiError::from(Error::InvalidInput(format!( - "invalid Repository key: {error}" - ))) - })?; - let RepositoryLogRead { - repository_key, - default_selector, - limit, - commits, - diagnostics, - } = repository_lookup( - api.live_repository_reader()? - .recent_log(&repository_key, query.limit), - )?; - Ok(Json(RepositoryLogResponse { - workspace_id: api.config.workspace_id, - repository_key, - default_selector, - limit, - items: commits, - diagnostics, - })) + api.store + .db_executor() + .run(move || { + server_api::validate_repository_key(&repository_key).map_err(|error| { + ApiError::from(Error::InvalidInput(format!( + "invalid Repository key: {error}" + ))) + })?; + let RepositoryLogRead { + repository_key, + default_selector, + limit, + commits, + diagnostics, + } = repository_lookup( + api.live_repository_reader()? + .recent_log(&repository_key, query.limit), + )?; + Ok(Json(RepositoryLogResponse { + workspace_id: api.config.workspace_id, + repository_key, + default_selector, + limit, + items: commits, + diagnostics, + })) + }) + .await } async fn list_hosts( State(api): State, ) -> ApiResult>> { - let limit = api.config.max_records.min(200); - let runtime_hosts = api.runtime.list_hosts(limit); - Ok(Json(InternalRuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtime_hosts.items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_hosts.diagnostics, - })) + host_io_executor()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtime_hosts = api.runtime.list_hosts(limit); + Ok(Json(InternalRuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtime_hosts.items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_hosts.diagnostics, + })) + }) + .await } async fn list_runtimes( State(api): State, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let runtimes = api.runtime.list_runtimes(limit); - Ok(Json(server_api::RuntimeListResponse { - workspace_id: api.config.workspace_id, - limit, - items: runtimes.items.into_iter().map(Into::into).collect(), - source: "worker_runtime_registry".to_string(), - diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), - })) + host_io_executor()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtimes = api.runtime.list_runtimes(limit); + Ok(Json(server_api::RuntimeListResponse { + workspace_id: api.config.workspace_id, + limit, + items: runtimes.items.into_iter().map(Into::into).collect(), + source: "worker_runtime_registry".to_string(), + diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), + })) + }) + .await } async fn list_workers( State(api): State, ) -> ApiResult> { - workers_response(api).map(Json) + api.store + .db_executor() + .run(move || workers_response(api).map(Json)) + .await } async fn create_remote_runtime( @@ -27317,6 +28761,16 @@ fn runtime_removal_config_guard( async fn execute_runtime_removal( api: &WorkspaceApi, operation: RuntimeRemovalOperation, +) -> ApiResult { + let api = api.clone(); + // Retain registration/commit recovery as one owned lifecycle workflow; + // SQL transaction boundaries and commit-before-Runtime order are unchanged. + run_host_workflow(async move { execute_runtime_removal_on_host(&api, operation).await }).await +} + +async fn execute_runtime_removal_on_host( + api: &WorkspaceApi, + operation: RuntimeRemovalOperation, ) -> ApiResult { if operation.state == RuntimeRemovalOperationState::Succeeded { return Ok(operation); @@ -27519,9 +28973,16 @@ async fn remove_remote_runtime( return Ok(Json(runtime_removal_response(operation))); } + let config_api = api.clone(); let config_state = api - .config_store - .load_workspace_config(&api.config.workspace_id)? + .store + .db_executor() + .run(move || { + config_api + .config_store + .load_workspace_config(&config_api.config.workspace_id) + }) + .await? .ok_or_else(|| { Error::RegistryInconsistency(format!( "Workspace {} has no active configuration", @@ -27602,7 +29063,10 @@ async fn test_runtime_connection( async fn get_worker_launch_options( State(api): State, ) -> ApiResult> { - Ok(Json(worker_launch_options_response(&api)?)) + api.store + .db_executor() + .run(move || Ok(Json(worker_launch_options_response(&api)?))) + .await } fn working_directory_request_from_repository( @@ -28418,30 +29882,44 @@ fn backend_resource_error_status(error: &BackendResourceError) -> StatusCode { async fn get_companion_status( State(api): State, ) -> ApiResult> { - Ok(Json(api.companion.status())) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.status()))) + .await } async fn get_companion_transcript( State(api): State, Query(query): Query, ) -> ApiResult> { - let limit = query.limit.unwrap_or(api.config.max_records).min(200); - let start = query.start.unwrap_or(0); - Ok(Json(api.companion.transcript(start, limit))) + api.store + .db_executor() + .run(move || { + let limit = query.limit.unwrap_or(api.config.max_records).min(200); + let start = query.start.unwrap_or(0); + Ok(Json(api.companion.transcript(start, limit))) + }) + .await } async fn post_companion_message( State(api): State, Json(request): Json, ) -> ApiResult> { - Ok(Json(api.companion.send_message(request))) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.send_message(request)))) + .await } async fn post_companion_cancel( State(api): State, Json(request): Json, ) -> ApiResult> { - Ok(Json(api.companion.cancel(request))) + api.store + .db_executor() + .run(move || Ok(Json(api.companion.cancel(request)))) + .await } fn runtime_contract_restore( @@ -28533,25 +30011,29 @@ async fn get_runtime_worker( State(api): State, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, ) -> ApiResult> { - let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let worker = api - .runtime - .worker(&worker_ref) - .map_err(|err| err.into_error())?; - let record = sync_worker_observation(&api, &worker)?; - let links = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; - let workdirs = api - .store - .list_workdir_registry(&api.config.workspace_id, 500)?; - let updated_at = record.updated_at.clone(); - let worker = - project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; - Ok(Json(server_api::RuntimeWorkerShowResponse { - worker, - updated_at, - })) + host_io_executor()? + .run(move || { + let worker_ref = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let worker = api + .runtime + .worker(&worker_ref) + .map_err(|err| err.into_error())?; + let record = sync_worker_observation(&api, &worker)?; + let links = api + .store + .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; + let workdirs = api + .store + .list_workdir_registry(&api.config.workspace_id, 500)?; + let updated_at = record.updated_at.clone(); + let worker = + project_worker_registry_projection(&api, Some(&worker), &record, links, &workdirs)?; + Ok(Json(server_api::RuntimeWorkerShowResponse { + worker, + updated_at, + })) + }) + .await } async fn restore_runtime_worker_with_context( @@ -28561,7 +30043,7 @@ async fn restore_runtime_worker_with_context( context: WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; let mut result = worker.restore(&context, request).await?; let projected_worker = if let Some(worker) = result.worker.as_ref() { // A replayed terminal receipt is historical evidence, not a new Runtime observation. @@ -28626,33 +30108,37 @@ async fn list_runtime_workers( AxumPath(runtime_id): AxumPath, Query(query): Query, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let (runtime_workers, source) = match query.status { - Some(RuntimeWorkersStatusFilter::Stopped) => ( - api.runtime - .list_stopped_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry_stopped", - ), - None => ( - api.runtime - .list_workers_for_runtime(&runtime_id, limit) - .map_err(|error| error.into_error())?, - "runtime_registry", - ), - }; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::RuntimeWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: source.to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) + host_io_executor()? + .run(move || { + let limit = api.config.max_records.min(200); + let (runtime_workers, source) = match query.status { + Some(RuntimeWorkersStatusFilter::Stopped) => ( + api.runtime + .list_stopped_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry_stopped", + ), + None => ( + api.runtime + .list_workers_for_runtime(&runtime_id, limit) + .map_err(|error| error.into_error())?, + "runtime_registry", + ), + }; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::RuntimeWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: source.to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) + }) + .await } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -29219,17 +30705,21 @@ async fn sync_runtime_config_bundle( AxumPath(runtime_id): AxumPath, Json(request): Json, ) -> ApiResult> { - let bundle = serde_json::from_value::(request.bundle) - .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; - let result = api - .runtime - .sync_config_bundle(&runtime_id, bundle) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) + host_io_executor()? + .run(move || { + let bundle = serde_json::from_value::(request.bundle) + .map_err(|error| invalid_contract_value("Runtime ConfigBundle", error))?; + let result = api + .runtime + .sync_config_bundle(&runtime_id, bundle) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) + }) + .await } async fn check_runtime_config_bundle( @@ -29237,21 +30727,25 @@ async fn check_runtime_config_bundle( AxumPath((runtime_id, bundle_id)): AxumPath<(String, String)>, Query(query): Query, ) -> ApiResult> { - let result = api - .runtime - .check_config_bundle( - &runtime_id, - ConfigBundleRef { - id: bundle_id, - digest: query.digest, - }, - ) - .map_err(|err| err.into_error())?; - Ok(Json(runtime_config_bundle_result_to_api( - result.state, - result.availability, - result.diagnostics, - )?)) + host_io_executor()? + .run(move || { + let result = api + .runtime + .check_config_bundle( + &runtime_id, + ConfigBundleRef { + id: bundle_id, + digest: query.digest, + }, + ) + .map_err(|err| err.into_error())?; + Ok(Json(runtime_config_bundle_result_to_api( + result.state, + result.availability, + result.diagnostics, + )?)) + }) + .await } #[cfg(test)] @@ -29271,7 +30765,7 @@ async fn dispatch_runtime_worker_input( worker_id: String, request: WorkerInputRequest, ) -> ApiResult> { - let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; + let worker = WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await?; let result = worker .input(&WorkerOperationContext::Backend, request) .await?; @@ -29282,11 +30776,15 @@ async fn delete_runtime_worker_uploaded_file( State(api): State, AxumPath((runtime_id, worker_id, artifact_id)): AxumPath<(String, String, String)>, ) -> ApiResult> { - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - api.runtime - .delete_worker_uploaded_file(&worker, &artifact_id) - .map_err(|err| err.into_error())?; - Ok(Json(WorkerFileDeleteResponse { deleted: true })) + host_io_executor()? + .run(move || { + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + api.runtime + .delete_worker_uploaded_file(&worker, &artifact_id) + .map_err(|err| err.into_error())?; + Ok(Json(WorkerFileDeleteResponse { deleted: true })) + }) + .await } async fn runtime_worker_completions( @@ -29294,13 +30792,17 @@ async fn runtime_worker_completions( AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, Json(request): Json, ) -> ApiResult> { - let request = worker_completions_request_from_api(request)?; - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let result = api - .runtime - .worker_completions(&worker, request) - .map_err(|err| err.into_error())?; - Ok(Json(worker_completions_result_to_api(result)?)) + host_io_executor()? + .run(move || { + let request = worker_completions_request_from_api(request)?; + let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; + let result = api + .runtime + .worker_completions(&worker, request) + .map_err(|err| err.into_error())?; + Ok(Json(worker_completions_result_to_api(result)?)) + }) + .await } fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource { @@ -29361,7 +30863,7 @@ async fn worker_protocol_ws( headers: HeaderMap, ws: WebSocketUpgrade, ) -> Response { - let worker = match WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) { + let worker = match WorkspaceWorker::resolve_async(&api, &runtime_id, &worker_id).await { Ok(worker) => worker, Err(error) => return ApiError::from(error).into_response(), }; @@ -29869,23 +31371,27 @@ async fn list_host_workers( State(api): State, AxumPath(host_id): AxumPath, ) -> ApiResult> { - let limit = api.config.max_records.min(200); - let runtime_workers = api - .runtime - .list_workers_for_host(&host_id, limit) - .map_err(|err| err.into_error())?; - let items = project_observed_workspace_workers(&api, runtime_workers.items)?; - Ok(Json(server_api::HostWorkerListResponse { - workspace_id: api.workspace_id().to_string(), - limit, - items, - source: "worker_runtime_registry".to_string(), - diagnostics: runtime_workers - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - })) + host_io_executor()? + .run(move || { + let limit = api.config.max_records.min(200); + let runtime_workers = api + .runtime + .list_workers_for_host(&host_id, limit) + .map_err(|err| err.into_error())?; + let items = project_observed_workspace_workers(&api, runtime_workers.items)?; + Ok(Json(server_api::HostWorkerListResponse { + workspace_id: api.workspace_id().to_string(), + limit, + items, + source: "worker_runtime_registry".to_string(), + diagnostics: runtime_workers + .diagnostics + .into_iter() + .map(Into::into) + .collect(), + })) + }) + .await } fn project_workspace_worker_with_attachments( @@ -32444,9 +33950,27 @@ impl From for ApiError { } } +impl From for server_api::RepositoryApiError { + fn from(error: Error) -> Self { + ApiError::from(error).into_repository_api_error() + } +} + impl From for ApiError { fn from(error: Error) -> Self { let diagnostics = match &error { + Error::DbCapacityExceeded + | Error::DbExecutorUnavailable + | Error::DbOperationOutcomeUnknown => vec![RuntimeDiagnostic { + code: match &error { + Error::DbCapacityExceeded => "db_capacity_exceeded", + Error::DbExecutorUnavailable => "db_executor_unavailable", + _ => "db_operation_outcome_unknown", + } + .into(), + severity: HostDiagnosticSeverity::Error, + message: error.to_string(), + }], Error::WorkdirSessionOpenFailed { transport, .. } => vec![RuntimeDiagnostic { code: "workdir_session_open_failed".to_string(), severity: HostDiagnosticSeverity::Error, @@ -32670,6 +34194,8 @@ fn api_error_status(error: &Error) -> StatusCode { { StatusCode::SERVICE_UNAVAILABLE } + Error::DbCapacityExceeded | Error::DbExecutorUnavailable => StatusCode::SERVICE_UNAVAILABLE, + Error::DbOperationOutcomeUnknown => StatusCode::SERVICE_UNAVAILABLE, Error::WorkspaceSigningIdentity { .. } => StatusCode::SERVICE_UNAVAILABLE, Error::WorkdirSessionOpenFailed { .. } | Error::RuntimeOperationFailed { .. } => { StatusCode::BAD_GATEWAY @@ -32727,6 +34253,10 @@ impl IntoResponse for ApiError { } } +#[cfg(test)] +#[path = "server_db_execution_tests.rs"] +mod db_execution_tests; + #[cfg(test)] mod tests { mod assignment_tracking_tests { @@ -33663,6 +35193,73 @@ mod tests { ); } + #[tokio::test] + async fn cancelled_workdir_db_waiter_retains_session_lock_until_mutation_finishes() { + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let api = test_api(dir.path()).await; + let worker = RuntimeWorkerRef::new("guard-runtime", "guard-worker"); + let lock = current_worker_session_lock(&api, &worker); + let guard = Arc::new(lock.clone().lock_owned().await); + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let record = WorkdirRegistryRecord { + workspace_id: TEST_WORKSPACE_ID.to_string(), + workdir_id: "cancelled-waiter-workdir".to_string(), + display_name: Some("Accepted mutation".to_string()), + source: WorkdirRegistrySource::Repository { + runtime_id: worker.runtime_id.clone(), + repository_id: api.config.repositories[0].id.clone(), + }, + creation_selector: None, + creation_ref: None, + creation_tree: None, + current_selector: None, + current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, + materialization_status: "present".to_string(), + cleanliness: "unknown".to_string(), + created_at: now.clone(), + updated_at: now, + }; + let (started, accepted) = tokio::sync::oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let stage_api = api.clone(); + let waiter = tokio::spawn(async move { + run_worker_workdir_db_stage_locked(&stage_api, &guard, move |api| { + let _ = started.send(()); + released + .recv_timeout(std::time::Duration::from_secs(5)) + .map_err(|error| Error::Store(error.to_string()))?; + api.store.upsert_workdir_registry(&record)?; + Ok::<_, Error>(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), accepted) + .await + .unwrap() + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + // This is the same Worker lock, not an executor-capacity proxy. + let still_locked = lock.try_lock().is_err(); + release.send(()).unwrap(); + assert!( + still_locked, + "accepted mutation lost its Worker session guard" + ); + let _guard = tokio::time::timeout(std::time::Duration::from_secs(5), lock.lock()) + .await + .expect("accepted mutation did not release its Worker lock"); + let persisted = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, "cancelled-waiter-workdir") + .unwrap() + .expect("accepted mutation was cancelled with its waiter"); + assert_eq!(persisted.display_name.as_deref(), Some("Accepted mutation")); + } + #[tokio::test] async fn backend_expiry_fences_offline_grant_and_persists_cleanup_completion() { let dir = tempfile::tempdir().unwrap(); @@ -33856,10 +35453,10 @@ mod tests { inject_external_workdir_cleanup_failure(&grant.workdir_id); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = session_lock.lock().await; + let session_guard = Arc::new(session_lock.lock_owned().await); let error = tokio::time::timeout( std::time::Duration::from_secs(1), - open_current_worker_workdir_session_locked(&api, &worker, &link), + open_current_worker_workdir_session_locked(&api, &worker, &link, &session_guard), ) .await .expect("session-open expiry cleanup must not reacquire its Worker lock") @@ -34404,10 +36001,11 @@ mod tests { ); let session_lock = current_worker_session_lock(&api, &worker_ref); - let session_guard = session_lock.lock().await; - let session = open_current_worker_workdir_session_locked(&api, &worker_ref, link) - .await - .unwrap(); + let session_guard = Arc::new(session_lock.lock_owned().await); + let session = + open_current_worker_workdir_session_locked(&api, &worker_ref, link, &session_guard) + .await + .unwrap(); drop(session_guard); assert_eq!(session.capabilities(), expected_capabilities); if index == 0 { @@ -34830,11 +36428,15 @@ mod tests { workdir::WorkdirSessionCapabilities::ALL ); let session_lock = current_worker_session_lock(&api, &worker); - let session_guard = session_lock.lock().await; - let broker_session = - open_current_worker_workdir_session_locked(&api, &worker, &external_link) - .await - .unwrap(); + let session_guard = Arc::new(session_lock.lock_owned().await); + let broker_session = open_current_worker_workdir_session_locked( + &api, + &worker, + &external_link, + &session_guard, + ) + .await + .unwrap(); assert_eq!( broker_session.capabilities(), workdir::WorkdirSessionCapabilities::ALL @@ -56217,7 +57819,7 @@ mod tests { ) .unwrap(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, other_workspace, @@ -56278,7 +57880,7 @@ mod tests { let proof = issue(); let source = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, TEST_WORKSPACE_ID, @@ -56296,7 +57898,7 @@ mod tests { .unwrap(); let proof = issue(); let result = crate::worker_source::verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + api.store.clone(), &api.config, &proof, TEST_WORKSPACE_ID, diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs index 0fcb072af..6aaa1e83c 100644 --- a/crates/workspace-server/src/server/subjektiv_job.rs +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -65,70 +65,83 @@ pub(super) fn require_candidate_grant( /// candidate decision. Once Idle is confirmed, stop cancels any legacy surface /// background work; the new Job regenerates it and does not infer surface success. pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult { + let job_api = api.clone(); + let subject_id = subject_id.to_owned(); let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); - let Some(owner) = api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - else { - return Ok(true); + let (worker, complete) = host_io_executor()? + .run(move || { + let api = &job_api; + let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); + let Some(owner) = api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + else { + return Ok((None, true)); + }; + let mut summary = api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())?; + summary.singleton_key = Some(owner.key.clone()); + require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; + let stopped = summary.state.eq_ignore_ascii_case("stopped"); + let idle = summary.state.eq_ignore_ascii_case("idle") + && summary + .worker_state + .as_ref() + .is_some_and(|s| s.state == protocol::WorkerState::Idle); + if !stopped && !idle { + return Ok((None, false)); + } + if api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .as_ref() + .map(|o| &o.worker) + != Some(&owner.worker) + { + return Err(Error::RepositoryConflict( + "legacy consolidation owner moved during cutover".into(), + ) + .into()); + } + if !summary.state.eq_ignore_ascii_case("stopped") { + let stopped = api + .runtime + .stop_worker( + &owner.worker, + WorkerLifecycleRequest { + reason: Some( + "Retire legacy subjektiv consolidator after foreground completion" + .into(), + ), + ticket_assignment: None, + }, + ) + .map_err(|e| e.into_error())?; + if stopped.state != InternalWorkerOperationState::Accepted { + return Ok((None, false)); + } + } + if !api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())? + .state + .eq_ignore_ascii_case("stopped") + { + return Ok((None, false)); + } + Ok::<_, ApiError>((Some(owner.worker), false)) + }) + .await?; + let Some(worker) = worker else { + return Ok(complete); }; - let mut summary = api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())?; - summary.singleton_key = Some(owner.key.clone()); - require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; - let stopped = summary.state.eq_ignore_ascii_case("stopped"); - let idle = summary.state.eq_ignore_ascii_case("idle") - && summary - .worker_state - .as_ref() - .is_some_and(|s| s.state == protocol::WorkerState::Idle); - if !stopped && !idle { - return Ok(false); - } - if api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - .as_ref() - .map(|o| &o.worker) - != Some(&owner.worker) - { - return Err(Error::RepositoryConflict( - "legacy consolidation owner moved during cutover".into(), - ) - .into()); - } - if !summary.state.eq_ignore_ascii_case("stopped") { - let stopped = api - .runtime - .stop_worker( - &owner.worker, - WorkerLifecycleRequest { - reason: Some( - "Retire legacy subjektiv consolidator after foreground completion".into(), - ), - ticket_assignment: None, - }, - ) - .map_err(|e| e.into_error())?; - if stopped.state != InternalWorkerOperationState::Accepted { - return Ok(false); - } - } - if !api - .runtime - .worker(&owner.worker) - .map_err(|e| e.into_error())? - .state - .eq_ignore_ascii_case("stopped") - { - return Ok(false); - } let removal = WorkerRemovalService::new(api) .execute_target_removal( api.runtime.as_ref(), - &owner.worker, + &worker, "subjektiv consolidation Job cutover", None, ) @@ -141,10 +154,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) - )) .into()); } - Ok(api - .store - .current_worker_singleton_owner(api.workspace_id(), &key)? - .is_none()) + let store = api.store.clone(); + let workspace_id = api.workspace_id().to_owned(); + api.store + .db_executor() + .run(move || { + store + .current_worker_singleton_owner(&workspace_id, &key) + .map(|owner| owner.is_none()) + .map_err(ApiError::from) + }) + .await } fn bounded_candidate_batch( diff --git a/crates/workspace-server/src/server/tests/auth_logging_tests.rs b/crates/workspace-server/src/server/tests/auth_logging_tests.rs index b4b51d7bc..b00d2e1b4 100644 --- a/crates/workspace-server/src/server/tests/auth_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/auth_logging_tests.rs @@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m &error, ) }); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert!( to_bytes(response.into_body(), usize::MAX) .await diff --git a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs index 7a4066b39..358b9a7bb 100644 --- a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs +++ b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs @@ -767,6 +767,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) { std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 }) ); assert!(execution.protocol_methods().is_empty()); + // SQLite jobs run on external threads; paused Tokio time cannot observe their + // progress and may auto-advance a reconnect timeout before SQL completes. + tokio::time::resume(); assert_same_worker_reconnect_and_stop(&worker, &context).await; } @@ -844,7 +847,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub } } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress() { let workspace = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs index 9bfbd3364..c71ea73df 100644 --- a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs @@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin let f = Fixture::new().await; let mut a = f.attach("1"); let (first, reply) = f.start_read("1", &mut a).await; - let mut second = Box::pin(f.app.clone().oneshot(f.operation("1"))); - // Poll through dispatch/auth until the handler waits on its resource lock. - assert!(futures::poll!(&mut second).is_pending()); + let request = f.operation("1"); + let proof = request + .headers() + .get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER) + .unwrap() + .to_str() + .unwrap(); + let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof) + .unwrap() + .jti; + let mut second = Box::pin(f.app.clone().oneshot(request)); + // DB auth is now genuinely asynchronous. Observe durable proof consumption, + // not a guessed number of polls, before advancing the injected proof clock. + let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap(); + tokio::time::timeout(BOUND, async { + loop { + assert!(futures::poll!(&mut second).is_pending()); + let consumed: bool = connection + .query_row( + "SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)", + [&jti], + |row| row.get(0), + ) + .unwrap(); + if consumed { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("second request never authenticated before resource wait"); assert!( a.try_recv().is_err(), "same session read escaped resource exclusion" @@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin assert_json(first.await.unwrap(), StatusCode::OK).await; assert!(futures::poll!(&mut second).is_pending()); let Some(ExternalProviderCommand::Operation { response, .. }) = - tokio::time::timeout(BOUND, a.recv()).await.unwrap() + tokio::time::timeout(BOUND, async { + tokio::select! { + command = a.recv() => command, + result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()), + } + }).await.unwrap() else { panic!("expected second read") }; @@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits( }), ); let mut deletion = Box::pin(f.app.clone().oneshot(request)); - assert!(futures::poll!(&mut deletion).is_pending()); let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await; - assert!( - gate.admit(false).is_none(), - "deletion start did not fence admission" - ); + tokio::time::timeout(BOUND, async { + loop { + assert!(futures::poll!(&mut deletion).is_pending()); + if gate.admit(false).is_none() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("deletion start did not fence admission after async authentication"); let session_id = session_store::new_session_id().to_string(); let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions"); let callback = f.request( diff --git a/crates/workspace-server/src/server/worker_operations.rs b/crates/workspace-server/src/server/worker_operations.rs index 35b60148f..ed38158bf 100644 --- a/crates/workspace-server/src/server/worker_operations.rs +++ b/crates/workspace-server/src/server/worker_operations.rs @@ -57,6 +57,19 @@ impl WorkerOperationContext { } Ok(()) } + pub(super) async fn from_request_async( + api: &WorkspaceApi, + context: &server_api::ServerRequestContext, + ) -> std::result::Result { + let owned_api = api.clone(); + let context = context.clone(); + api.store + .db_executor() + .run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context))) + .await + .map_err(|error| ApiError::from(error).into_repository_api_error())? + } + pub(super) fn from_request( api: &WorkspaceApi, context: &server_api::ServerRequestContext, @@ -162,8 +175,20 @@ impl WorkspaceWorker { source: crate::worker_source::VerifiedWorkerMutationSource, reason: &str, ) -> std::result::Result { - self.current_record() - .map_err(|error| error.error.to_string())?; + let removal = self.removal.clone(); + let identity = self.identity.clone(); + self.removal + .store + .db_executor() + .run(move || { + removal + .store + .get_worker_registry(&removal.workspace_id, &identity)? + .ok_or_else(|| Error::UnknownWorker { worker: identity }) + .map(drop) + }) + .await + .map_err(|error| error.to_string())?; self.removal .execute_async( source, @@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor } impl WorkspaceWorker { + pub(crate) async fn resolve_async( + api: &WorkspaceApi, + runtime_id: &str, + reference: &str, + ) -> Result { + let owned_api = api.clone(); + let runtime_id = runtime_id.to_owned(); + let reference = reference.to_owned(); + api.store + .db_executor() + .run(move || Self::resolve(&owned_api, &runtime_id, &reference)) + .await + } + pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result { Self::resolve_with_services( api.clone(), @@ -239,7 +278,36 @@ impl WorkspaceWorker { context: &WorkerOperationContext, permission: &str, ) -> ApiResult>> { - self.current_record()?; + let worker = self.clone(); + let initial_context = context.clone(); + let permission = permission.to_owned(); + let initial_permission = permission.clone(); + let grant = self + .api + .store + .db_executor() + .run(move || -> ApiResult> { + worker.current_record()?; + let WorkerOperationContext::WorkerControl { controller } = initial_context else { + return Ok(None); + }; + if worker + .api + .store + .get_worker_registry(worker.api.workspace_id(), &controller)? + .is_none() + { + return Err(Error::UnknownWorker { worker: controller }.into()); + } + Ok(Some(authorize_known_worker_permission( + &worker.api, + worker.api.workspace_id(), + &controller, + &worker.identity, + &initial_permission, + )?)) + }) + .await?; match context { #[cfg(test)] WorkerOperationContext::Backend => Ok(None), @@ -248,42 +316,35 @@ impl WorkspaceWorker { Ok(None) } WorkerOperationContext::WorkerControl { controller } => { - if self - .api - .store - .get_worker_registry(self.api.workspace_id(), controller)? - .is_none() - { - return Err(Error::UnknownWorker { - worker: controller.clone(), - } - .into()); - } - let grant = authorize_known_worker_permission( - &self.api, - self.api.workspace_id(), - controller, - &self.identity, - permission, - )?; + let grant = grant.expect("Worker control context resolved a grant"); + // Await admission outside the DB lane; the recheck owns the guard + // until it finishes, even if its async waiter is cancelled. let guard = worker_control_lock(&self.api, &grant.grant_id) .lock_owned() .await; - self.current_record()?; - let current_grant = authorize_known_worker_permission( - &self.api, - self.api.workspace_id(), - controller, - &self.identity, - permission, - )?; - if current_grant.grant_id != grant.grant_id { - return Err(Error::RepositoryConflict( - "Worker control grant changed while waiting for admission".into(), - ) - .into()); - } - Ok(Some(guard)) + let worker = self.clone(); + let controller = controller.clone(); + self.api + .store + .db_executor() + .run(move || -> ApiResult<_> { + worker.current_record()?; + let current_grant = authorize_known_worker_permission( + &worker.api, + worker.api.workspace_id(), + &controller, + &worker.identity, + &permission, + )?; + if current_grant.grant_id != grant.grant_id { + return Err(Error::RepositoryConflict( + "Worker control grant changed while waiting for admission".into(), + ) + .into()); + } + Ok(Some(guard)) + }) + .await } } } @@ -293,8 +354,14 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: server_api::WorkerRestoreRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "restore").await?; - self.restore_internal(request) + let guard = self.authorize_operation(context, "restore").await?; + let worker = self.clone(); + host_io_executor()? + .run(move || { + let _guard = guard; + worker.restore_internal(request) + }) + .await } /// Trusted Backend callers still supply an explicit observed intent. Runtime @@ -312,27 +379,43 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "stop").await?; - let result = self - .api - .runtime - .stop_worker(&self.identity, request) - .map_err(|e| e.into_error())?; + let guard = self.authorize_operation(context, "stop").await?; + let worker = self.clone(); + let (result, guard) = host_io_executor()? + .run(move || -> ApiResult<_> { + let result = worker + .api + .runtime + .stop_worker(&worker.identity, request) + .map_err(|e| e.into_error())?; + Ok((result, guard)) + }) + .await?; if result.state != InternalWorkerOperationState::Accepted { return Ok(result); } parse_runtime_worker_id_for_registry(&self.identity.worker_id)?; let session_lock = current_worker_session_lock(&self.api, &self.identity); - let _session_guard = session_lock.lock().await; + let session_guard = session_lock.lock_owned().await; close_current_worker_session_locked(&self.api, &self.identity).await?; - if let Some(record) = self - .api - .store - .get_worker_registry(self.api.workspace_id(), &self.identity)? - { - sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?; - } - Ok(result) + let worker = self.clone(); + host_io_executor()? + .run(move || -> ApiResult<_> { + let (_control, _session) = (guard, session_guard); + if let Some(record) = worker + .api + .store + .get_worker_registry(worker.api.workspace_id(), &worker.identity)? + { + sync_linked_workdir_after_worker_stop( + &worker.api, + &worker.identity.runtime_id, + &record, + )?; + } + Ok(result) + }) + .await } pub(super) async fn cancel( @@ -340,12 +423,18 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { - let _guard = self.authorize_operation(context, "cancel").await?; - Ok(self - .api - .runtime - .cancel_worker(&self.identity, request) - .map_err(|e| e.into_error())?) + let guard = self.authorize_operation(context, "cancel").await?; + let worker = self.clone(); + host_io_executor()? + .run(move || -> ApiResult<_> { + let _guard = guard; + Ok(worker + .api + .runtime + .cancel_worker(&worker.identity, request) + .map_err(|e| e.into_error())?) + }) + .await } pub(super) async fn input( @@ -358,46 +447,59 @@ impl WorkspaceWorker { } else { "send_input" }; - let _guard = self.authorize_operation(context, permission).await?; + let guard = self.authorize_operation(context, permission).await?; let session_lock = current_worker_session_lock(&self.api, &self.identity); - let _session_guard = session_lock.lock().await; - self.ensure_interactive()?; + let session_guard = session_lock.lock_owned().await; + self.ensure_interactive().await?; if matches!(context, WorkerOperationContext::WorkerControl { .. }) && request.kind == WorkerInputKind::User { request.kind = WorkerInputKind::UserIfIdle; } - Ok(self - .api - .runtime - .send_input(&self.identity, request) - .map_err(|e| e.into_error())?) + let worker = self.clone(); + host_io_executor()? + .run(move || -> ApiResult<_> { + let (_control, _session) = (guard, session_guard); + Ok(worker + .api + .runtime + .send_input(&worker.identity, request) + .map_err(|e| e.into_error())?) + }) + .await } - fn ensure_interactive(&self) -> ApiResult<()> { - if let Some(claim) = self - .api + async fn ensure_interactive(&self) -> ApiResult<()> { + let worker = self.clone(); + self.api .store - .get_pending_ticket_assignment_operation_for_worker( - self.api.workspace_id(), - &self.identity, - )? - { - return Err(Error::TicketAssignmentConflict(format!( - "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", - claim.operation_id, - )).into()); - } - if self - .api - .store - .worker_registry_projection(self.api.workspace_id(), &self.identity)? - .and_then(|projection| projection.job) - .is_some() - { - return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); - } - Ok(()) + .db_executor() + .run(move || -> ApiResult<()> { + if let Some(claim) = worker + .api + .store + .get_pending_ticket_assignment_operation_for_worker( + worker.api.workspace_id(), + &worker.identity, + )? + { + return Err(Error::TicketAssignmentConflict(format!( + "Worker has pending Ticket claim {}; reconcile that exact operation before admitting input", + claim.operation_id, + )).into()); + } + if worker + .api + .store + .worker_registry_projection(worker.api.workspace_id(), &worker.identity)? + .and_then(|projection| projection.job) + .is_some() + { + return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); + } + Ok(()) + }) + .await } pub(super) async fn set_pinned( @@ -405,33 +507,44 @@ impl WorkspaceWorker { context: &WorkerOperationContext, pinned: bool, ) -> ApiResult { - let _guard = self.authorize_operation(context, "pin").await?; - let retention_state = if pinned { "pinned" } else { "normal" }; - let changed = self.api.worker_projection.publish_ordered(|| { - self.api - .store - .update_worker_retention( - self.api.workspace_id(), - &self.identity, - retention_state, - now_registry_timestamp().as_str(), - ) - .map(|commit| (!commit.changes.is_empty(), commit)) - })?; - if !changed { - return Err(cleanup_api_error( - &self.identity.runtime_id, - "workspace_worker_retention_unknown_worker", - "Worker is not known to the Backend registry", - )); - } - Ok(WorkerRetentionResponse { - workspace_id: self.api.workspace_id().to_owned(), - runtime_id: self.identity.runtime_id.clone(), - worker_id: self.identity.worker_id.clone(), - pinned, - retention_state: retention_state.into(), - }) + let guard = self.authorize_operation(context, "pin").await?; + let worker = self.clone(); + self.api + .store + .db_executor() + .run(move || -> ApiResult { + // Publication and its DB mutation are one owned stage. Accepted + // work retains control admission through commit and publication. + let _guard = guard; + let retention_state = if pinned { "pinned" } else { "normal" }; + let changed = worker.api.worker_projection.publish_ordered(|| { + worker + .api + .store + .update_worker_retention( + worker.api.workspace_id(), + &worker.identity, + retention_state, + now_registry_timestamp().as_str(), + ) + .map(|commit| (!commit.changes.is_empty(), commit)) + })?; + if !changed { + return Err(cleanup_api_error( + &worker.identity.runtime_id, + "workspace_worker_retention_unknown_worker", + "Worker is not known to the Backend registry", + )); + } + Ok(WorkerRetentionResponse { + workspace_id: worker.api.workspace_id().to_owned(), + runtime_id: worker.identity.runtime_id.clone(), + worker_id: worker.identity.worker_id.clone(), + pinned, + retention_state: retention_state.into(), + }) + }) + .await } /// The caller owns the full cleanup-plan digest check. The removal service @@ -611,7 +724,7 @@ impl WorkspaceWorkerMethodSender { .await?; let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity); let _session_guard = session_lock.lock().await; - self.worker.ensure_interactive()?; + self.worker.ensure_interactive().await?; let WorkerOperationContext::Browser { source, .. } = context else { return Err(Error::WorkspacePermissionDenied( "Browser Worker protocol context required".into(), diff --git a/crates/workspace-server/src/server/workspace_admission.rs b/crates/workspace-server/src/server/workspace_admission.rs index 06c94b9a8..5f8335629 100644 --- a/crates/workspace-server/src/server/workspace_admission.rs +++ b/crates/workspace-server/src/server/workspace_admission.rs @@ -72,6 +72,28 @@ pub(super) struct RequestLease { callback: bool, } +impl crate::db_executor::DbOperationLifetime for RequestLease { + fn retain(&self) -> Box { + // This is continuation of an already admitted request, not fresh admission. + // Retain its drain responsibility before transferring work to the DB owner; + // dropping/aborting the HTTP waiter must not allow deletion past live SQL. + let mut state = self + .gate + .state + .lock() + .expect("Workspace admission poisoned"); + if self.callback { + state.callbacks += 1; + } else { + state.requests += 1; + } + Box::new(RequestLease { + gate: self.gate.clone(), + callback: self.callback, + }) + } +} + impl Drop for RequestLease { fn drop(&mut self) { let mut state = self diff --git a/crates/workspace-server/src/server_db_execution_tests.rs b/crates/workspace-server/src/server_db_execution_tests.rs new file mode 100644 index 000000000..22052925a --- /dev/null +++ b/crates/workspace-server/src/server_db_execution_tests.rs @@ -0,0 +1,606 @@ +//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs). +use super::*; +use crate::SqliteWorkspaceStore; +use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime}; +use rusqlite::{Connection, ffi}; +use std::{ + ffi::c_void, + future::Future, + num::NonZeroUsize, + sync::{atomic::AtomicBool, mpsc}, + thread::JoinHandle, + time::Duration as StdDuration, +}; +use tokio::sync::oneshot; + +const WORKSPACE: &str = "db-execution"; +const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects"; +const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2); +const WATCHDOG: StdDuration = StdDuration::from_secs(5); + +async fn observed(future: impl Future) -> T { + tokio::time::timeout(OBSERVE_LIMIT, future) + .await + .expect("request did not reach the expected DB/lifecycle state") +} + +struct Fixture { + // Keep temporary authority files alive until all other fixture owners drop. + server: WorkspaceServerApi, + api: WorkspaceApi, + executor: DbExecutor, + _dir: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + // Explicit startup config: never resolve the developer's live data dir. + let config = ServerConfig { + workspace_id: WORKSPACE.into(), + workspace_display_name: "DB execution fixture".into(), + workspace_created_at: "2026-01-01T00:00:00Z".into(), + workspace_execution_root: dir.path().into(), + database_path: dir.path().join("server.db"), + data_root: dir.path().join("data"), + embedded_runtime_store_root: dir.path().join("runtime"), + static_assets_dir: None, + auth: AuthConfig::Passkey { + rp_id: "localhost".into(), + origin: "http://localhost:5173".into(), + public_base_url: "http://localhost:5173".into(), + cookie_name: "db_execution_session".into(), + }, + max_records: 100, + repositories: Vec::new(), + runtime_event_sources: Vec::new(), + remote_runtime_sources: Vec::new(), + backend_base_url: Some("http://localhost:8787".into()), + }; + let store = Arc::new( + SqliteWorkspaceStore::open(&config.database_path) + .unwrap() + .with_db_executor(executor.clone()), + ); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + store + .upsert_user(&UserRecord { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + created_at: config.workspace_created_at.clone(), + updated_at: config.workspace_created_at.clone(), + }) + .unwrap(); + store + .create_browser_session(&crate::store::BrowserSessionRecord { + session_id: "fixture-session".into(), + token_hash: crate::auth::token_hash("fixture-browser-token"), + user_id: "fixture-user".into(), + created_at: config.workspace_created_at.clone(), + expires_at: "2999-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + let api = WorkspaceApi::new(config.clone(), store.clone()) + .await + .unwrap(); + let server = WorkspaceServerApi::new(config, store); + // Cache only the real inner router; avoid starting the unrelated hook. + server + .apis + .lock() + .await + .insert(WORKSPACE.into(), api.clone()); + server + .routers + .lock() + .await + .insert(WORKSPACE.into(), build_inner_router(api.clone())); + executor.drain().await.unwrap(); + Self { + server, + api, + executor, + _dir: dir, + } + } + + fn request(method: &str, path: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header("cookie", "db_execution_session=fixture-browser-token") + .header(ORIGIN, "http://localhost:5173") + .body(axum::body::Body::from(body.to_string())) + .unwrap() + } + + fn subject_request(role: &str) -> Request { + Self::request( + "POST", + SUBJECTS, + serde_json::json!({ "role": role, "behavior_md": "" }), + ) + } + + fn dispatch(&self, request: Request) -> tokio::task::JoinHandle { + let server = self.server.clone(); + tokio::spawn(async move { dispatch_workspace_request(State(server), request).await }) + } + + // A pre-authenticated generated router isolates handler admission from the + // earlier browser-session DB read. Full authentication/dispatch is exercised + // separately by the cancellation and warm-Feature tests below. + fn inner_router(&self) -> Router { + build_inner_router(self.api.clone()).layer(Extension(RequestActor { + user_id: "fixture-user".into(), + account_id: format!("account-{WORKSPACE}"), + handle: "fixture-user".into(), + display_name: "Fixture user".into(), + auth_method: ActorAuthMethod::BrowserSession, + })) + } + + fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) { + let database = self + .api + .feature_storage + .open(&self.api.subjektiv_registration) + .unwrap(); + // Initialize the trusted scope, then obtain SQLite's actual file path + // from the managed handle rather than guessing the Feature directory. + crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap(); + let path = database + .with_connection(|conn| { + Ok(PathBuf::from(conn.query_row( + "SELECT file FROM pragma_database_list WHERE name = 'main'", + [], + |row| row.get::<_, String>(0), + )?)) + }) + .unwrap(); + (database, path) + } +} + +/// SQLite write ownership lives on an OS thread, including a bounded watchdog. +/// Drop releases and joins it even when an assertion panics on the async side. +struct WriteLock { + release: Option>, + thread: Option>, + released: Arc, +} + +impl WriteLock { + fn acquire(path: PathBuf) -> Self { + let (ready, held) = mpsc::sync_channel(1); + let (release, permit) = mpsc::channel(); + let released = Arc::new(AtomicBool::new(false)); + let flag = released.clone(); + let thread = std::thread::spawn(move || { + let conn = Connection::open(path).unwrap(); + conn.execute_batch("BEGIN IMMEDIATE").unwrap(); + ready.send(()).unwrap(); + let _ = permit.recv_timeout(WATCHDOG); + conn.execute_batch("ROLLBACK").unwrap(); + flag.store(true, Ordering::SeqCst); + }); + held.recv_timeout(WATCHDOG) + .expect("SQLite lock holder did not acquire its transaction"); + Self { + release: Some(release), + thread: Some(thread), + released, + } + } + + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} +impl Drop for WriteLock { + fn drop(&mut self) { + self.release(); + } +} + +struct BusyGate { + entered: Option>, + resume: mpsc::Receiver<()>, +} + +/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's +/// userdata API can, without process globals or changing production hooks. +struct BusyObservation { + database: crate::FeatureDatabase, + gate: Box, + resume: Option>, +} +impl BusyObservation { + fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) { + unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 { + // SAFETY: owned Box remains registered until Drop unregisters it, + // serialized by the managed connection mutex. + let gate = unsafe { &mut *context.cast::() }; + if let Some(entered) = gate.entered.take() { + let _ = entered.send(()); + return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok()); + } + 0 + } + let (entered, waiting) = oneshot::channel(); + let (resume, permit) = mpsc::channel(); + let mut gate = Box::new(BusyGate { + entered: Some(entered), + resume: permit, + }); + database + .with_connection(|conn| { + // SAFETY: handle belongs to this connection; the Box is stable. + let code = unsafe { + ffi::sqlite3_busy_handler( + conn.handle(), + Some(busy), + (&mut *gate as *mut BusyGate).cast(), + ) + }; + assert_eq!(code, ffi::SQLITE_OK); + Ok(()) + }) + .unwrap(); + ( + Self { + database, + gate, + resume: Some(resume), + }, + waiting, + ) + } + fn resume(&mut self) { + if let Some(resume) = self.resume.take() { + let _ = resume.send(()); + } + } +} +impl Drop for BusyObservation { + fn drop(&mut self) { + self.resume(); + self.database + .with_connection(|conn| { + // SAFETY: unregister before the boxed userdata is freed. + unsafe { + ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut()); + } + Ok(()) + }) + .unwrap(); + let _ = &self.gate; + } +} + +// A handler with one DB stage can expose its admission without adding a test +// hook to production. Polling its future to Pending proves run accepted it. +struct AdmissionObservation(Mutex>>); +impl DbOperationLifetime for AdmissionObservation { + fn retain(&self) -> Box { + if let Some(accepted) = self.0.lock().unwrap().take() { + let _ = accepted.send(()); + } + Box::new(()) + } +} +fn admission_observation() -> (Arc, oneshot::Receiver<()>) { + let (accepted, receiver) = oneshot::channel(); + ( + Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))), + receiver, + ) +} + +struct StateCheckSaturation { + executor: DbExecutor, + stages: std::sync::atomic::AtomicUsize, + releases: Arc>>>, +} +impl DbOperationLifetime for StateCheckSaturation { + fn retain(&self) -> Box { + if self.stages.fetch_add(1, Ordering::SeqCst) == 1 { + // Authentication has completed; saturate just before the lifecycle + // lookup is admitted. The callback's own dispatch admissions are + // distinct stages and do not recurse into this branch. + let (started, running) = mpsc::channel(); + let (release, permit) = mpsc::channel(); + self.executor + .dispatch(move || { + let _ = started.send(()); + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + running + .recv_timeout(WATCHDOG) + .expect("state-check blocker did not start"); + self.releases.lock().unwrap().push(release); + let (release, permit) = mpsc::channel(); + self.executor + .dispatch(move || { + let _ = permit.recv_timeout(WATCHDOG); + }) + .unwrap(); + self.releases.lock().unwrap().push(release); + } + Box::new(()) + } +} +struct ReleaseStateCheckBlockers(Arc>>>); +impl Drop for ReleaseStateCheckBlockers { + fn drop(&mut self) { + self.0.lock().unwrap().clear(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() { + let fixture = Fixture::new().await; + let releases = Arc::new(Mutex::new(Vec::new())); + let release = ReleaseStateCheckBlockers(releases.clone()); + let source = Arc::new(StateCheckSaturation { + executor: fixture.executor.clone(), + stages: std::sync::atomic::AtomicUsize::new(0), + releases, + }); + let router = Router::new() + .route( + SUBJECTS, + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .layer(axum::middleware::from_fn_with_state( + fixture.api.clone(), + authorize_workspace_api_request, + )); + let response = observed(crate::db_executor::with_operation_lifetime( + source, + router.oneshot(Fixture::subject_request("not-executed")), + )) + .await + .unwrap(); + drop(release); + observed(fixture.executor.drain()).await.unwrap(); + let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await; + assert!( + !body.to_string().contains("Workspace is deleting"), + "{body}" + ); +} + +async fn runtime_progress_while_locked(lock: &WriteLock) { + let control = tokio::spawn(async { "control progressed" }); + assert_eq!(observed(control).await.unwrap(), "control progressed"); + // The timer is the contract, not a guessed wait for request completion. + observed(tokio::time::sleep(StdDuration::from_millis(1))).await; + assert!( + !lock.released.load(Ordering::SeqCst), + "SQLite watchdog released the lock before runtime progress" + ); +} +async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value { + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); + serde_json::from_slice(&bytes).expect("typed JSON response") +} +fn subject_count(path: &Path, role: &str) -> i64 { + Connection::open(path) + .unwrap() + .query_row( + "SELECT count(*) FROM subjects WHERE role = ?1", + [role], + |row| row.get(0), + ) + .unwrap() +} + +#[tokio::test(flavor = "current_thread")] +async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() { + let fixture = Fixture::new().await; + let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone()); + let (lifetime, accepted) = admission_observation(); + let request = Fixture::request( + "POST", + "/api/w/db-execution/tickets", + serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(), + ); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture.inner_router().oneshot(request), + )); + assert!(futures::poll!(&mut response).is_pending()); + observed(accepted).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + futures::poll!(&mut response).is_pending(), + "mutation responded before the SQLite lock was released" + ); + let reader = Connection::open(&fixture.api.config.database_path).unwrap(); + let count: i64 = reader + .query_row( + "SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1", + [WORKSPACE], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 0, "Ticket must not be visible before commit"); + lock.release(); + let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; + assert!(body["id"].is_string(), "{body}"); + // Ticket dispatch can call Runtime callbacks, so its operation owner is + // the separate bounded host executor, not the store executor. + observed(host_io_executor().unwrap().drain()).await.unwrap(); + fixture.executor.drain().await.unwrap(); + let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap(); + assert_eq!(count, 1, "one accepted HTTP mutation must commit once"); +} + +#[tokio::test(flavor = "current_thread")] +async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + drop(database); // FeatureStorage retains Weak handles: the route must cold-open. + let mut lock = WriteLock::acquire(path); + let (lifetime, accepted) = admission_observation(); + let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null); + let mut response = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture.inner_router().oneshot(request), + )); + assert!(futures::poll!(&mut response).is_pending()); + observed(accepted).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + futures::poll!(&mut response).is_pending(), + "cold open must wait for its SQLite scope transaction" + ); + lock.release(); + let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await; + assert_eq!(body["items"], serde_json::json!([]), "{body}"); + fixture.executor.drain().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let response = fixture.dispatch(Fixture::subject_request("locked-subject")); + observed(waiting).await.unwrap(); + runtime_progress_while_locked(&lock).await; + assert!( + !response.is_finished(), + "HTTP response must not publish an uncommitted subject" + ); + assert_eq!(subject_count(&path, "locked-subject"), 0); + lock.release(); + busy.resume(); + let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await; + assert_eq!(body["role"], "locked-subject", "{body}"); + assert!( + body["memory_fingerprint"].is_string(), + "response assembly must finish in the DB operation: {body}" + ); + fixture.executor.drain().await.unwrap(); + assert_eq!(subject_count(&path, "locked-subject"), 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let response = fixture.dispatch(Fixture::subject_request("cancelled-subject")); + observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling. + response.abort(); + assert!(observed(response).await.unwrap_err().is_cancelled()); + let admission = fixture.server.admission_for_workspace(WORKSPACE).await; + let mut deletion = Box::pin(admission.lock_deletion()); + assert!( + futures::poll!(&mut deletion).is_pending(), + "aborting HTTP must not drop the accepted SQL RequestLease" + ); + assert!( + admission.admit(false).is_none(), + "deletion must fence fresh requests" + ); + assert_eq!(subject_count(&path, "cancelled-subject"), 0); + lock.release(); + busy.resume(); + fixture.executor.drain().await.unwrap(); + let deletion = observed(deletion).await; + assert_eq!( + subject_count(&path, "cancelled-subject"), + 1, + "cancelled waiter must neither cancel nor retry accepted SQL" + ); + drop(deletion); + assert!( + admission.admit(false).is_some(), + "deletion lease cleanup must reopen admission" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() { + let fixture = Fixture::new().await; + let (database, path) = fixture.feature_database(); + let mut lock = WriteLock::acquire(path.clone()); + let (mut busy, waiting) = BusyObservation::install(database); + let running = fixture.dispatch(Fixture::subject_request("running-subject")); + observed(waiting).await.unwrap(); + let (lifetime, accepted) = admission_observation(); + let mut queued = Box::pin(crate::db_executor::with_operation_lifetime( + lifetime, + fixture + .inner_router() + .oneshot(Fixture::subject_request("queued-subject")), + )); + assert!(futures::poll!(&mut queued).is_pending()); + observed(accepted).await.unwrap(); + drop(queued); // Accepted job still owns the only waiting slot. + let rejected = observed( + fixture + .inner_router() + .oneshot(Fixture::subject_request("rejected-subject")), + ) + .await + .unwrap(); + let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await; + let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap(); + assert!( + error.message.contains("capacity"), + "typed overload error: {body}" + ); + assert_eq!(subject_count(&path, "queued-subject"), 0); + assert_eq!(subject_count(&path, "rejected-subject"), 0); + let mut drained = Box::pin(fixture.executor.drain()); + assert!(futures::poll!(&mut drained).is_pending()); + lock.release(); + busy.resume(); + response_json(observed(running).await.unwrap(), StatusCode::CREATED).await; + observed(drained).await.unwrap(); + assert_eq!( + subject_count(&path, "queued-subject"), + 1, + "cancelled accepted mutation persists exactly once" + ); + assert_eq!( + subject_count(&path, "rejected-subject"), + 0, + "overload must not execute rejected SQL" + ); + let retry = fixture.dispatch(Fixture::subject_request("after-drain")); + response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await; + fixture.executor.drain().await.unwrap(); + assert_eq!( + subject_count(&path, "after-drain"), + 1, + "drain must release actual-job capacity" + ); +} diff --git a/crates/workspace-server/src/server_drive.rs b/crates/workspace-server/src/server_drive.rs index 57c79ef7f..33bb2e908 100644 --- a/crates/workspace-server/src/server_drive.rs +++ b/crates/workspace-server/src/server_drive.rs @@ -51,14 +51,13 @@ async fn blocking( mutation: bool, action: impl FnOnce() -> workspace_drive::Result + Send + 'static, ) -> DriveResult { - tokio::task::spawn_blocking(action) + crate::db_executor::DbExecutor::shared() + .run(move || Ok::<_, Error>(action())) .await - .map_err(|_| { - if mutation { - DriveApiError::new(DriveApiErrorCode::OutcomeUnknown) - } else { - failure(503) - } + .map_err(|error| match error { + Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503), + _ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown), + _ => failure(503), })? .map_err(|error| domain_error(error, mutation)) } diff --git a/crates/workspace-server/src/server_drive_grants.rs b/crates/workspace-server/src/server_drive_grants.rs index 9d85eaf67..4d1f82fe5 100644 --- a/crates/workspace-server/src/server_drive_grants.rs +++ b/crates/workspace-server/src/server_drive_grants.rs @@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into) -> DriveApiError { }) } fn mutation_error(error: Error) -> DriveApiError { - if matches!(error, Error::Sqlite(_) | Error::Store(_)) { + if matches!( + error, + Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown + ) { fail(DriveApiErrorCode::OutcomeUnknown) } else { pre_error(error) @@ -97,23 +100,27 @@ pub(super) async fn authorized_drive( }; // Reject missing/read-only grants before upload body reception. This check // is admission only: every later operation rechecks under its own lock. - tokio::task::spawn_blocking(move || { - drive.check_authorized(write)?; - Ok::<_, workspace_drive::Error>(drive) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? - .map_err(|error| { - fail(match error { - workspace_drive::Error::Denied => DriveApiErrorCode::Denied, - workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { - DriveApiErrorCode::Conflict - } - workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, - workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, - _ => DriveApiErrorCode::StorageUnavailable, + api.store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + drive.check_authorized(write)?; + Ok::<_, workspace_drive::Error>(drive) + })()) + }) + .await + .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? + .map_err(|error| { + fail(match error { + workspace_drive::Error::Denied => DriveApiErrorCode::Denied, + workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => { + DriveApiErrorCode::Conflict + } + workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound, + workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid, + _ => DriveApiErrorCode::StorageUnavailable, + }) }) - }) } pub(super) async fn create_grant( @@ -152,9 +159,10 @@ pub(super) async fn create_grant( revoked_at: None, }; let store = api.store.clone(); - tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant)) + api.store + .db_executor() + .run(move || store.create_workspace_drive_grant(&grant)) .await - .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? .map_err(mutation_error) } @@ -175,13 +183,12 @@ pub(super) async fn revoke_grant( actor.account_id.clone(), ); let store = api.store.clone(); - tokio::task::spawn_blocking(move || { - store.revoke_workspace_drive_grant(&workspace, &grant, &actor) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))? - .map_err(mutation_error)? - .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) + api.store + .db_executor() + .run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor)) + .await + .map_err(mutation_error)? + .ok_or_else(|| fail(DriveApiErrorCode::NotFound)) } fn validate_grant_id(id: &str) -> DriveResult<()> { @@ -212,12 +219,14 @@ pub(super) async fn list_grants( } let workspace = workspace_id.to_owned(); let store = api.store.clone(); - let mut grants = tokio::task::spawn_blocking(move || { - store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) - }) - .await - .map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))? - .map_err(pre_error)?; + let mut grants = api + .store + .db_executor() + .run(move || { + store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref()) + }) + .await + .map_err(pre_error)?; let more = grants.len() > limit; grants.truncate(limit); let next_after = if more { diff --git a/crates/workspace-server/src/server_workspace_config.rs b/crates/workspace-server/src/server_workspace_config.rs index 0e0a54959..dbdb24e6c 100644 --- a/crates/workspace-server/src/server_workspace_config.rs +++ b/crates/workspace-server/src/server_workspace_config.rs @@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into) -> Failure { fn mutation_error(error: Error) -> Failure { // Storage/service failures at a mutation boundary may include a commit // acknowledgement failure. Never describe those as a guaranteed rollback. - if matches!(&error, Error::Sqlite(_) | Error::Store(_)) { + if matches!( + &error, + Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown + ) { unknown() } else { pre_error(error) @@ -186,9 +189,23 @@ pub(super) async fn get( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> ConfigResult> { - let _guard = lock(api, worker).await?; - let grant = grant_for_worker(api, worker)?; - Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true))) + let guard = lock(api, worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let worker = worker.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + let grant = grant_for_worker(&api, &worker)?; + Ok::<_, Failure>( + active_link(&api, &worker, &grant, None)? + .map(|link| attachment(&grant, &link, true)), + ) + })()) + }) + .await + .map_err(pre_error)? } pub(super) async fn attach( api: &WorkspaceApi, @@ -198,46 +215,59 @@ pub(super) async fn attach( if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) { return Err(fail(400, "invalid_alias")); } - let _guard = lock(api, worker).await?; - let grant = grant_for_worker(api, worker)?; - let access = request.access.unwrap_or(grant.access); - if access == Access::ReadWrite && grant.access != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - if let Some(mut link) = active_link(api, worker, &grant, None)? { - // Same active alias/identity is idempotent; explicit write never silently downgrades. - if request.access == Some(Access::ReadWrite) - && effective_access(&grant, &link) != Access::ReadWrite - { - return Err(fail(403, "access_denied")); - } - if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly { - link.capabilities = access_capabilities(access); - link = api - .store - .attach_worker_workdir(&link) - .map_err(mutation_error)?; - } - return Ok(attachment(&grant, &link, true)); - } - let link = api - .store - .attach_worker_workdir(&WorkerWorkdirLinkRecord { - connection_id: String::new(), - workspace_id: grant.workspace_id.clone(), - worker: worker.clone(), - workdir_id: grant.working_directory_id.clone(), - alias: ALIAS.into(), - capabilities: access_capabilities(access), - linked_at: now_registry_timestamp(), - unlinked_at: None, + let guard = lock(api, worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let worker = worker.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + let grant = grant_for_worker(&api, &worker)?; + let access = request.access.unwrap_or(grant.access); + if access == Access::ReadWrite && grant.access != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + if let Some(mut link) = active_link(&api, &worker, &grant, None)? { + // Same active alias/identity is idempotent; explicit write never silently downgrades. + if request.access == Some(Access::ReadWrite) + && effective_access(&grant, &link) != Access::ReadWrite + { + return Err(fail(403, "access_denied")); + } + if access == Access::ReadOnly + && effective_access(&grant, &link) != Access::ReadOnly + { + link.capabilities = access_capabilities(access); + link = api + .store + .attach_worker_workdir(&link) + .map_err(mutation_error)?; + } + return Ok(attachment(&grant, &link, true)); + } + let link = api + .store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: grant.workspace_id.clone(), + worker: worker.clone(), + workdir_id: grant.working_directory_id.clone(), + alias: ALIAS.into(), + capabilities: access_capabilities(access), + linked_at: now_registry_timestamp(), + unlinked_at: None, + }) + .map_err(mutation_error)?; + // No Runtime filesystem/session synchronization for a logical config target. + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(attachment(&grant, &link, false)) + })()) }) - .map_err(mutation_error)?; - // No Runtime filesystem/session synchronization for a logical config target. - api.worker_projection - .refresh(worker) - .map_err(|_| unknown())?; - Ok(attachment(&grant, &link, false)) + .await + .map_err(mutation_error)? } fn resolve( api: &WorkspaceApi, @@ -313,9 +343,22 @@ pub(super) async fn observe( for path in &request.paths { path_valid(path, true)?; } - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, _guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -427,9 +470,22 @@ pub(super) async fn read( request: WorkspaceConfigReadRequest, ) -> ConfigResult { path_valid(&request.path, false)?; - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, _guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -482,12 +538,25 @@ pub(super) async fn commit( server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?, } } - let _guard = lock(api, worker).await?; - let (grant, link) = resolve(api, worker, &request.connection_id)?; - if effective_access(&grant, &link) != Access::ReadWrite { - return Err(fail(403, "access_denied")); - } - let state = tree(api)?; + let guard = lock(api, worker).await?; + let resolution_api = api.clone(); + let resolution_worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let (grant, link, state, guard) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?; + if effective_access(&grant, &link) != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + let state = tree(&resolution_api)?; + Ok::<_, Failure>((grant, link, state, guard)) + })()) + }) + .await + .map_err(pre_error)??; let schema_fingerprint = api .config_schema_registry .compose() @@ -501,17 +570,19 @@ pub(super) async fn commit( // Evaluation/semantic validation are pure preparation. Bound this stage // without ever dispatching persistence in a timed-out background task. let preparation_api = api.clone(); - let preparation = tokio::task::spawn_blocking(move || { - prepare_workspace_config_tree( + let executor = api.store.db_executor(); + let preparation = executor.run(move || { + // Timeout/cancellation releases exclusion only after actual preparation ends. + let candidate = prepare_workspace_config_tree( &preparation_api, &preparation_api.config.workspace_id, &canonical, - ) + )?; + Ok::<_, Error>((candidate, guard)) }); - let candidate = tokio::time::timeout(DEADLINE, preparation) + let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation) .await .map_err(|_| fail(503, "evaluation_timeout"))? - .map_err(|_| fail(500, "evaluation_unavailable"))? .map_err(pre_error)?; if candidate.base_digest != state.snapshot.digest || candidate.base_toolchain_fingerprint != state.contract.fingerprint @@ -520,15 +591,78 @@ pub(super) async fn commit( { return Err(fail(409, "stale_validator")); } - // Recheck durable authority before the first auxiliary/canonical effects. - let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?; - if current_grant != grant || current_link.capabilities != link.capabilities { - return Err(fail(409, "authority_changed")); + let persistence_api = api.clone(); + let worker = worker.clone(); + let connection_id = request.connection_id.clone(); + let expected_grant = grant.clone(); + let expected_link = link.clone(); + let (state, publication) = api + .store + .db_executor() + .run(move || { + Ok::<_, Error>((|| { + let (current_grant, current_link) = + resolve(&persistence_api, &worker, &connection_id)?; + if current_grant != expected_grant + || current_link.capabilities != expected_link.capabilities + { + return Err(fail(409, "authority_changed")); + } + // Preserve the shared helper's ordering and atomic CAS, but never + // run its remote Runtime publication on the sole DB worker. + project_repository_access_candidate( + &*persistence_api.store, + &persistence_api.repository_secrets, + &persistence_api.config.workspace_id, + &candidate, + ) + .map_err(|_| unknown())?; + let state = persistence_api + .config_store + .commit_evaluated_workspace_config( + &persistence_api.config.workspace_id, + &candidate, + ) + .map_err(|_| unknown())?; + let publication = persistence_api + .prompt_projection_cache + .resolve(&persistence_api.config.workspace_id, &state) + .ok() + .and_then(|projection| { + // Dispatch from the actual persistence owner even if its + // waiter was cancelled. Runtime I/O owns exclusion, not + // a DB worker or an open SQL transaction. + let (complete, publication) = tokio::sync::oneshot::channel(); + let runtime = persistence_api.runtime.clone(); + let admission = host_io_executor().and_then(|executor| { + executor.dispatch(move || { + let _guard = guard; + let _diagnostics = runtime + .observe_workspace_prompt_projection((*projection).clone()); + let _ = complete.send(()); + }) + }); + match admission { + Ok(()) => Some(publication), + Err(error) => { + tracing::warn!( + workspace_id = %persistence_api.config.workspace_id, + %error, + "Workspace config committed but Runtime publication was not admitted" + ); + None + } + } + }); + Ok((state, publication)) + })()) + }) + .await + .map_err(mutation_error)??; + if let Some(publication) = publication { + // Publication is best effort and cannot turn an acknowledged CAS into an error. + let _ = publication.await; } - // From this boundary repository credential projection may have side effects. - // Never falsely claim rollback or automatically replay a failed/uncertain save. - let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate) - .map_err(|_| unknown())?; Ok(WorkspaceConfigCommitResponse { validator: validator(&grant, &link, &state, &schema_fingerprint, ""), digest: state.snapshot.digest, @@ -544,37 +678,59 @@ pub(super) async fn create_grant( .await .map_err(pre_error)?; let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id); - let _guard = lock(api, &worker).await?; - let summary = api - .runtime - .worker(&worker) - .map_err(|_| fail(404, "worker_not_found"))?; + let guard = lock(api, &worker).await?; + let runtime = api.runtime.clone(); + let lookup_worker = worker.clone(); + let (summary, guard) = host_io_executor() + .map_err(pre_error)? + .run(move || { + Ok::<_, Error>((|| { + let summary = runtime + .worker(&lookup_worker) + .map_err(|_| fail(404, "worker_not_found"))?; + Ok::<_, Failure>((summary, guard)) + })()) + }) + .await + .map_err(pre_error)??; if summary.workspace.workspace_id.as_deref() != Some(workspace_id) { return Err(fail(403, "access_denied")); } - if let Some(grant) = api - .store - .current_workspace_config_grant(workspace_id, &worker) - .map_err(pre_error)? - { - if grant.access != request.access { - return Err(fail(409, "grant_exists_revoke_first")); - } - return Ok(grant); - } - let grant = WorkspaceConfigGrantResponse { - grant_id: Uuid::now_v7().to_string(), - workspace_id: workspace_id.into(), - runtime_id: worker.runtime_id, - worker_id: worker.worker_id, - working_directory_id: Uuid::now_v7().to_string(), - access: request.access, - revoked: false, - }; - api.store - .create_workspace_config_grant(&grant, &actor.account_id) - .map_err(mutation_error)?; - Ok(grant) + let executor = api.store.db_executor(); + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let account_id = actor.account_id.clone(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + if let Some(grant) = api + .store + .current_workspace_config_grant(&workspace_id, &worker) + .map_err(pre_error)? + { + if grant.access != request.access { + return Err(fail(409, "grant_exists_revoke_first")); + } + return Ok(grant); + } + let grant = WorkspaceConfigGrantResponse { + grant_id: Uuid::now_v7().to_string(), + workspace_id, + runtime_id: worker.runtime_id, + worker_id: worker.worker_id, + working_directory_id: Uuid::now_v7().to_string(), + access: request.access, + revoked: false, + }; + api.store + .create_workspace_config_grant(&grant, &account_id) + .map_err(mutation_error)?; + Ok(grant) + })()) + }) + .await + .map_err(mutation_error)? } pub(super) async fn revoke_grant( api: &WorkspaceApi, @@ -585,39 +741,60 @@ pub(super) async fn revoke_grant( require_workspace_owner(api, workspace_id, actor, "Workspace config grants") .await .map_err(pre_error)?; + let lookup_api = api.clone(); + let lookup_workspace_id = workspace_id.to_owned(); + let lookup_grant_id = grant_id.to_owned(); let mut grant = api .store - .get_workspace_config_grant(workspace_id, grant_id) + .db_executor() + .run(move || { + lookup_api + .store + .get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id) + }) + .await .map_err(pre_error)? .ok_or_else(|| fail(404, "not_found"))?; let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id); - let _guard = lock(api, &worker).await?; - api.store - .revoke_workspace_config_grant(workspace_id, grant_id) - .map_err(mutation_error)?; - grant.revoked = true; - // Revocation is durable before unlink; a failed cleanup cannot restore access. - if let Some(link) = api - .store - .list_workdir_worker_links(workspace_id, &grant.working_directory_id) - .map_err(|_| unknown())? - .into_iter() - .find(|link| link.unlinked_at.is_none() && link.worker == worker) - { - api.store - .detach_worker_workdir_connection( - workspace_id, - &worker, - &link.alias, - &link.connection_id, - &now_registry_timestamp(), - ) - .map_err(|_| unknown())?; - } - api.worker_projection - .refresh(&worker) - .map_err(|_| unknown())?; - Ok(grant) + let guard = lock(api, &worker).await?; + let executor = api.store.db_executor(); + let api = api.clone(); + let workspace_id = workspace_id.to_owned(); + let grant_id = grant_id.to_owned(); + executor + .run(move || { + let _guard = guard; + Ok::<_, Error>((|| { + api.store + .revoke_workspace_config_grant(&workspace_id, &grant_id) + .map_err(mutation_error)?; + grant.revoked = true; + // Revocation is durable before unlink; a failed cleanup cannot restore access. + if let Some(link) = api + .store + .list_workdir_worker_links(&workspace_id, &grant.working_directory_id) + .map_err(|_| unknown())? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.worker == worker) + { + api.store + .detach_worker_workdir_connection( + &workspace_id, + &worker, + &link.alias, + &link.connection_id, + &now_registry_timestamp(), + ) + .map_err(|_| unknown())?; + } + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(grant) + })()) + }) + .await + .map_err(mutation_error)? } pub(super) fn summary( @@ -655,7 +832,11 @@ pub(super) fn summary( mod tests { use super::*; #[test] - fn workspace_config_mutation_storage_failures_are_conservatively_unknown() { + fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() { + assert_eq!( + mutation_error(Error::DbOperationOutcomeUnknown).classification, + Classification::Unknown + ); assert_eq!( mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification, Classification::Unknown diff --git a/crates/workspace-server/src/server_workspace_config_tests.rs b/crates/workspace-server/src/server_workspace_config_tests.rs index ac11e7766..72fb9e910 100644 --- a/crates/workspace-server/src/server_workspace_config_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_tests.rs @@ -803,9 +803,9 @@ mod workspace_config_integration { .supports(workdir::WorkdirSessionCapability::Command) ); let lock = current_worker_session_lock(api, &fixture.worker); - let _guard = lock.lock().await; + let _guard = Arc::new(lock.lock_owned().await); assert!( - open_current_worker_workdir_session_locked(api, &fixture.worker, &link) + open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard) .await .is_err() ); diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index c88faeec8..b5cfd5cec 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -1123,6 +1123,10 @@ impl WorkspaceResourceKind { #[async_trait] pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { + fn db_executor(&self) -> crate::db_executor::DbExecutor { + crate::db_executor::DbExecutor::shared() + } + async fn schema_version(&self) -> Result; fn resource_key( &self, @@ -2179,6 +2183,7 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { #[derive(Clone)] pub struct SqliteWorkspaceStore { conn: Arc>, + db_executor: crate::db_executor::DbExecutor, } impl SqliteWorkspaceStore { @@ -2222,9 +2227,20 @@ impl SqliteWorkspaceStore { prepare_connection(&conn)?; Ok(Self { conn: Arc::new(Mutex::new(conn)), + db_executor: crate::db_executor::DbExecutor::shared(), }) } + /// Override execution limits for a hermetic fixture; subsequent clones share it. + pub fn with_db_executor(mut self, executor: crate::db_executor::DbExecutor) -> Self { + self.db_executor = executor; + self + } + + pub fn db_executor(&self) -> crate::db_executor::DbExecutor { + self.db_executor.clone() + } + pub(crate) fn with_conn(&self, f: impl FnOnce(&Connection) -> Result) -> Result { let conn = self .conn @@ -3794,8 +3810,16 @@ fn insert_workspace_runtime_binding_audit( #[async_trait] impl ControlPlaneStore for SqliteWorkspaceStore { + fn db_executor(&self) -> crate::db_executor::DbExecutor { + self.db_executor.clone() + } + async fn schema_version(&self) -> Result { - self.with_conn(current_schema_version) + let store = self.clone(); + + self.db_executor + .run(move || store.with_conn(current_schema_version)) + .await } fn resource_key( @@ -3927,100 +3951,114 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()> { - self.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - #[cfg(test)] - if record.owner_account_id == "owner-account" { - // Unrelated store tests use this explicit fixture identity. Keep the - // production owner contract strict while giving those fixtures a real - // User Account row instead of reviving ownerless Workspace setup. - tx.execute( - "INSERT OR IGNORE INTO accounts ( - account_id, kind, handle, display_name, created_at, updated_at - ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", - params![record.owner_account_id.as_str(), record.created_at.as_str()], - )?; - } - let owner_kind = tx - .query_row( - "SELECT kind FROM accounts WHERE account_id = ?1", - params![record.owner_account_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if owner_kind.as_deref() != Some("user") { - return Err(Error::Store(format!( - "Workspace owner `{}` must reference an existing User Account", - record.owner_account_id - ))); - } - let current_owner = tx - .query_row( - "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", - params![record.workspace_id.as_str()], - |row| row.get::<_, String>(0), - ) - .optional()?; - if current_owner - .as_deref() - .is_some_and(|owner| owner != record.owner_account_id) - { - return Err(Error::Store(format!( - "Workspace `{}` owner is immutable through upsert", - record.workspace_id - ))); - } - tx.execute( - r#"INSERT INTO workspaces ( - workspace_id, owner_account_id, display_name, state, created_at, updated_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(workspace_id) DO UPDATE SET - display_name = excluded.display_name, - state = excluded.state, - updated_at = excluded.updated_at"#, - params![ - record.workspace_id, - record.owner_account_id, - record.display_name, - record.state, - record.created_at, - record.updated_at, - ], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_memory_settings ( - workspace_id, language, created_at, updated_at - ) VALUES (?1, 'English', ?2, ?3)"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.execute( - r#"INSERT OR IGNORE INTO workspace_signing_identities ( - workspace_id, key_id, algorithm, public_key, public_key_fingerprint, - private_material_ref, state, created_at, provisioned_at, updated_at - ) VALUES ( - ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, - 'workspace-signing/' || ?1 || '/ed25519-v1', - 'pending_provisioning', ?2, NULL, ?3 - )"#, - params![record.workspace_id, record.created_at, record.updated_at], - )?; - tx.commit()?; - Ok(()) - })?; - self.materialize_workspace_config(&record.workspace_id, &record.created_at) + let store = self.clone(); + let record = record.clone(); + self.db_executor + .run(move || { + let store = &store; + let record = &record; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + #[cfg(test)] + if record.owner_account_id == "owner-account" { + // Unrelated store tests use this explicit fixture identity. Keep the + // production owner contract strict while giving those fixtures a real + // User Account row instead of reviving ownerless Workspace setup. + tx.execute( + "INSERT OR IGNORE INTO accounts ( + account_id, kind, handle, display_name, created_at, updated_at + ) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)", + params![record.owner_account_id.as_str(), record.created_at.as_str()], + )?; + } + let owner_kind = tx + .query_row( + "SELECT kind FROM accounts WHERE account_id = ?1", + params![record.owner_account_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if owner_kind.as_deref() != Some("user") { + return Err(Error::Store(format!( + "Workspace owner `{}` must reference an existing User Account", + record.owner_account_id + ))); + } + let current_owner = tx + .query_row( + "SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1", + params![record.workspace_id.as_str()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if current_owner + .as_deref() + .is_some_and(|owner| owner != record.owner_account_id) + { + return Err(Error::Store(format!( + "Workspace `{}` owner is immutable through upsert", + record.workspace_id + ))); + } + tx.execute( + r#"INSERT INTO workspaces ( + workspace_id, owner_account_id, display_name, state, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(workspace_id) DO UPDATE SET + display_name = excluded.display_name, + state = excluded.state, + updated_at = excluded.updated_at"#, + params![ + record.workspace_id, + record.owner_account_id, + record.display_name, + record.state, + record.created_at, + record.updated_at, + ], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_memory_settings ( + workspace_id, language, created_at, updated_at + ) VALUES (?1, 'English', ?2, ?3)"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.execute( + r#"INSERT OR IGNORE INTO workspace_signing_identities ( + workspace_id, key_id, algorithm, public_key, public_key_fingerprint, + private_material_ref, state, created_at, provisioned_at, updated_at + ) VALUES ( + ?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL, + 'workspace-signing/' || ?1 || '/ed25519-v1', + 'pending_provisioning', ?2, NULL, ?3 + )"#, + params![record.workspace_id, record.created_at, record.updated_at], + )?; + tx.commit()?; + Ok(()) + })?; + store.materialize_workspace_config(&record.workspace_id, &record.created_at) + }) + .await } async fn get_workspace(&self, workspace_id: &str) -> Result> { - self.with_conn(|conn| { - conn.query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional() - .map_err(Error::from) - }) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor.run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + store.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional() + .map_err(Error::from) + }) + }).await } async fn update_workspace_display_name( @@ -4029,58 +4067,68 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_updated_at: &str, display_name: &str, ) -> Result> { - validate_identifier("workspace_id", workspace_id)?; - validate_non_empty("expected_updated_at", expected_updated_at)?; - validate_non_empty("display_name", display_name)?; - self.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let current = tx - .query_row( - r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at - FROM workspaces WHERE workspace_id = ?1"#, - params![workspace_id], - read_workspace_record, - ) - .optional()?; - let Some(current) = current else { - tx.commit()?; - return Ok(None); - }; - if current.updated_at != expected_updated_at { - tx.commit()?; - return Ok(None); - } - if current.display_name == display_name { - tx.commit()?; - return Ok(Some(current)); - } + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let expected_updated_at = expected_updated_at.to_owned(); + let display_name = display_name.to_owned(); + self.db_executor.run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let expected_updated_at = expected_updated_at.as_str(); + let display_name = display_name.as_str(); + validate_identifier("workspace_id", workspace_id)?; + validate_non_empty("expected_updated_at", expected_updated_at)?; + validate_non_empty("display_name", display_name)?; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current = tx + .query_row( + r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1"#, + params![workspace_id], + read_workspace_record, + ) + .optional()?; + let Some(current) = current else { + tx.commit()?; + return Ok(None); + }; + if current.updated_at != expected_updated_at { + tx.commit()?; + return Ok(None); + } + if current.display_name == display_name { + tx.commit()?; + return Ok(Some(current)); + } - let now = chrono::Utc::now(); - let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) - .ok() - .map(|previous| previous.with_timezone(&chrono::Utc)) - .filter(|previous| *previous >= now) - .map(|previous| previous + chrono::Duration::nanoseconds(1)) - .unwrap_or(now) - .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); - let changed = tx.execute( - r#"UPDATE workspaces - SET display_name = ?3, updated_at = ?4 - WHERE workspace_id = ?1 AND updated_at = ?2"#, - params![workspace_id, expected_updated_at, display_name, updated_at], - )?; - if changed != 1 { + let now = chrono::Utc::now(); + let updated_at = chrono::DateTime::parse_from_rfc3339(¤t.updated_at) + .ok() + .map(|previous| previous.with_timezone(&chrono::Utc)) + .filter(|previous| *previous >= now) + .map(|previous| previous + chrono::Duration::nanoseconds(1)) + .unwrap_or(now) + .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + let changed = tx.execute( + r#"UPDATE workspaces + SET display_name = ?3, updated_at = ?4 + WHERE workspace_id = ?1 AND updated_at = ?2"#, + params![workspace_id, expected_updated_at, display_name, updated_at], + )?; + if changed != 1 { + tx.commit()?; + return Ok(None); + } + let updated = WorkspaceRecord { + display_name: display_name.to_string(), + updated_at, + ..current + }; tx.commit()?; - return Ok(None); - } - let updated = WorkspaceRecord { - display_name: display_name.to_string(), - updated_at, - ..current - }; - tx.commit()?; - Ok(Some(updated)) - }) + Ok(Some(updated)) + }) + }).await } fn create_workspace_bootstrap( @@ -4600,7 +4648,17 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result> { - SqliteWorkspaceStore::get_workspace_runtime_binding(self, workspace_id, runtime_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::get_workspace_runtime_binding(store, workspace_id, runtime_id) + }) + .await } async fn list_workspace_runtime_bindings( @@ -4608,7 +4666,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, include_revoked: bool, ) -> Result> { - SqliteWorkspaceStore::list_workspace_runtime_bindings(self, workspace_id, include_revoked) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + SqliteWorkspaceStore::list_workspace_runtime_bindings( + store, + workspace_id, + include_revoked, + ) + }) + .await } async fn has_other_active_workspace_runtime_binding( @@ -4616,11 +4686,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( - self, - workspace_id, - runtime_id, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::has_other_active_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + ) + }) + .await } async fn delete_workspace_runtime_binding( @@ -4628,7 +4708,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id: &str, runtime_id: &str, ) -> Result { - SqliteWorkspaceStore::delete_workspace_runtime_binding(self, workspace_id, runtime_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::delete_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + ) + }) + .await } async fn reserve_runtime_removal( @@ -4640,22 +4734,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: &str, config_digest: &str, ) -> Result { - SqliteWorkspaceStore::reserve_runtime_removal( - self, - workspace_id, - runtime_id, - operation_id, - request_fingerprint, - expected_binding_id, - config_digest, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let operation_id = operation_id.to_owned(); + let request_fingerprint = request_fingerprint.to_owned(); + let expected_binding_id = expected_binding_id.to_owned(); + let config_digest = config_digest.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let operation_id = operation_id.as_str(); + let request_fingerprint = request_fingerprint.as_str(); + let expected_binding_id = expected_binding_id.as_str(); + let config_digest = config_digest.as_str(); + SqliteWorkspaceStore::reserve_runtime_removal( + store, + workspace_id, + runtime_id, + operation_id, + request_fingerprint, + expected_binding_id, + config_digest, + ) + }) + .await } async fn get_runtime_removal( &self, operation_id: &str, ) -> Result> { - SqliteWorkspaceStore::get_runtime_removal(self, operation_id) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::get_runtime_removal(store, operation_id) + }) + .await } async fn mark_runtime_removal_failed( @@ -4663,14 +4783,36 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, failure_category: &str, ) -> Result { - SqliteWorkspaceStore::mark_runtime_removal_failed(self, operation_id, failure_category) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + let failure_category = failure_category.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + let failure_category = failure_category.as_str(); + SqliteWorkspaceStore::mark_runtime_removal_failed( + store, + operation_id, + failure_category, + ) + }) + .await } async fn commit_runtime_binding_removal( &self, operation_id: &str, ) -> Result { - SqliteWorkspaceStore::commit_runtime_binding_removal(self, operation_id) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::commit_runtime_binding_removal(store, operation_id) + }) + .await } async fn complete_runtime_removal( @@ -4678,18 +4820,34 @@ impl ControlPlaneStore for SqliteWorkspaceStore { operation_id: &str, runtime_registration_removed: bool, ) -> Result { - SqliteWorkspaceStore::complete_runtime_removal( - self, - operation_id, - runtime_registration_removed, - ) + let store = self.clone(); + let operation_id = operation_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let operation_id = operation_id.as_str(); + SqliteWorkspaceStore::complete_runtime_removal( + store, + operation_id, + runtime_registration_removed, + ) + }) + .await } async fn list_resumable_runtime_removals( &self, workspace_id: &str, ) -> Result> { - SqliteWorkspaceStore::list_resumable_runtime_removals(self, workspace_id) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + SqliteWorkspaceStore::list_resumable_runtime_removals(store, workspace_id) + }) + .await } async fn upsert_workspace_runtime_binding_record( @@ -4697,7 +4855,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record: WorkspaceRuntimeBinding, replace: bool, ) -> Result { - SqliteWorkspaceStore::upsert_workspace_runtime_binding(self, record, replace) + let store = self.clone(); + + self.db_executor + .run(move || { + let store = &store; + SqliteWorkspaceStore::upsert_workspace_runtime_binding(store, record, replace) + }) + .await } async fn revoke_workspace_runtime_binding_record( @@ -4706,12 +4871,24 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, revoked_at: &str, ) -> Result { - SqliteWorkspaceStore::revoke_workspace_runtime_binding( - self, - workspace_id, - runtime_id, - revoked_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let revoked_at = revoked_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let revoked_at = revoked_at.as_str(); + SqliteWorkspaceStore::revoke_workspace_runtime_binding( + store, + workspace_id, + runtime_id, + revoked_at, + ) + }) + .await } async fn put_workspace_runtime_binding_key( @@ -4720,12 +4897,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore { expected_binding_id: Option<&str>, actor_account_id: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - SqliteWorkspaceStore::put_workspace_runtime_binding_key( - self, - record, - expected_binding_id, - actor_account_id, - ) + let store = self.clone(); + let expected_binding_id = expected_binding_id.map(str::to_owned); + let actor_account_id = actor_account_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let expected_binding_id = expected_binding_id.as_deref(); + let actor_account_id = actor_account_id.as_str(); + SqliteWorkspaceStore::put_workspace_runtime_binding_key( + store, + record, + expected_binding_id, + actor_account_id, + ) + }) + .await } async fn update_workspace_runtime_binding_metadata( @@ -4736,14 +4923,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore { base_url: &str, updated_at: &str, ) -> Result { - SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( - self, - workspace_id, - runtime_id, - display_name, - base_url, - updated_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let display_name = display_name.to_owned(); + let base_url = base_url.to_owned(); + let updated_at = updated_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let display_name = display_name.as_str(); + let base_url = base_url.as_str(); + let updated_at = updated_at.as_str(); + SqliteWorkspaceStore::update_workspace_runtime_binding_metadata( + store, + workspace_id, + runtime_id, + display_name, + base_url, + updated_at, + ) + }) + .await } async fn revoke_workspace_runtime_binding_key( @@ -4754,14 +4957,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore { actor_account_id: &str, revoked_at: &str, ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { - SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( - self, - workspace_id, - runtime_id, - expected_binding_id, - actor_account_id, - revoked_at, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let expected_binding_id = expected_binding_id.to_owned(); + let actor_account_id = actor_account_id.to_owned(); + let revoked_at = revoked_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let expected_binding_id = expected_binding_id.as_str(); + let actor_account_id = actor_account_id.as_str(); + let revoked_at = revoked_at.as_str(); + SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( + store, + workspace_id, + runtime_id, + expected_binding_id, + actor_account_id, + revoked_at, + ) + }) + .await } async fn list_workspace_runtime_binding_audit( @@ -4770,12 +4989,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore { runtime_id: &str, limit: usize, ) -> Result> { - SqliteWorkspaceStore::list_workspace_runtime_binding_audit( - self, - workspace_id, - runtime_id, - limit, - ) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + SqliteWorkspaceStore::list_workspace_runtime_binding_audit( + store, + workspace_id, + runtime_id, + limit, + ) + }) + .await } async fn consume_worker_mutation_source_jti( @@ -4787,21 +5016,35 @@ impl ControlPlaneStore for SqliteWorkspaceStore { now_seconds: u64, consumed_at: &str, ) -> Result { - self.with_conn(|conn| { - let transaction = conn.unchecked_transaction()?; - transaction.execute( - "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", - params![now_seconds], - )?; - let inserted = transaction.execute( - r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( - workspace_id, runtime_id, jti, expires_at, consumed_at - ) VALUES (?1, ?2, ?3, ?4, ?5)"#, - params![workspace_id, runtime_id, jti, expires_at, consumed_at], - )?; - transaction.commit()?; - Ok(inserted == 1) - }) + let store = self.clone(); + let workspace_id = workspace_id.to_owned(); + let runtime_id = runtime_id.to_owned(); + let jti = jti.to_owned(); + let consumed_at = consumed_at.to_owned(); + self.db_executor + .run(move || { + let store = &store; + let workspace_id = workspace_id.as_str(); + let runtime_id = runtime_id.as_str(); + let jti = jti.as_str(); + let consumed_at = consumed_at.as_str(); + store.with_conn(|conn| { + let transaction = conn.unchecked_transaction()?; + transaction.execute( + "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", + params![now_seconds], + )?; + let inserted = transaction.execute( + r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( + workspace_id, runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, ?2, ?3, ?4, ?5)"#, + params![workspace_id, runtime_id, jti, expires_at, consumed_at], + )?; + transaction.commit()?; + Ok(inserted == 1) + }) + }) + .await } fn plan_worker_removal( diff --git a/crates/workspace-server/src/worker_projection.rs b/crates/workspace-server/src/worker_projection.rs index ed910de04..cb6a10dde 100644 --- a/crates/workspace-server/src/worker_projection.rs +++ b/crates/workspace-server/src/worker_projection.rs @@ -5,7 +5,7 @@ use chrono::Utc; use protocol::subscription::{ EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot, }; -use tokio::sync::broadcast; +use tokio::sync::{Semaphore, broadcast}; use tokio::task::JoinHandle; use crate::runtime_subscription::{ @@ -59,7 +59,14 @@ pub struct WorkerProjectionService { events: broadcast::Sender, ordering: Mutex<()>, attach_gate: tokio::sync::Mutex<()>, - tasks: Mutex>>, + tasks: Mutex>, +} + +struct RuntimeProjectionLifetime { + consumer: Option>, + // The permit belongs to the actual DB closure, not its cancellable async waiter. + // Keep this owner in the map even if attachment initialization is cancelled. + operations: Arc, } impl WorkerProjectionService { @@ -75,17 +82,89 @@ impl WorkerProjectionService { }) } + /// Stops consumers but does not claim that accepted database work has finished. + /// Async shutdown callers must use `shutdown_async` to drain accepted operations. pub fn shutdown(&self) { - for (_, task) in self + for lifetime in self .tasks .lock() .expect("Worker projection task lock poisoned") - .drain() + .values() { - task.abort(); + if let Some(task) = &lifetime.consumer { + task.abort(); + } } } + pub async fn shutdown_async(&self) { + let _attach = self.attach_gate.lock().await; + self.shutdown(); + let runtime_ids = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .keys() + .cloned() + .collect::>(); + for runtime_id in runtime_ids { + self.drain_runtime(&runtime_id).await; + } + self.tasks + .lock() + .expect("Worker projection task lock poisoned") + .clear(); + } + + async fn drain_runtime(&self, runtime_id: &str) { + let previous = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .get_mut(runtime_id) + .map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone())); + if let Some((consumer, operations)) = previous { + if let Some(consumer) = consumer { + consumer.abort(); + let _ = consumer.await; + } + // Aborting the waiter cannot abort a queued/running synchronous job. + // Cancellation here leaves the gate in the map for the next drain. + let _drained = operations + .acquire_owned() + .await + .expect("operation gate closed"); + } + } + + // Caller holds attach_gate. Install ownership before any initialization await. + async fn replace_runtime(&self, runtime_id: &str) -> Arc { + self.drain_runtime(runtime_id).await; + let operations = Arc::new(Semaphore::new(1)); + self.tasks + .lock() + .expect("Worker projection task lock poisoned") + .insert( + runtime_id.to_string(), + RuntimeProjectionLifetime { + consumer: None, + operations: operations.clone(), + }, + ); + operations + } + + pub async fn subscribe_ordered_async( + self: &Arc, + limit: usize, + ) -> crate::Result { + let service = self.clone(); + self.store + .db_executor() + .run(move || service.subscribe_ordered(limit)) + .await + } + pub fn subscribe_ordered(&self, limit: usize) -> crate::Result { let _ordering = self .ordering @@ -125,19 +204,10 @@ impl WorkerProjectionService { broker: &RuntimeSubscriptionBroker, runtime_id: &str, ) -> Result<(), RuntimeSubscriptionBrokerError> { - // Serialize replacement and wait for the old consumer to stop before the new - // lifetime can apply its snapshot. Late notifications can therefore never - // cross a Runtime ownership boundary. + // Serialize replacement, stop admission by aborting the consumer, and then + // drain ownership held by the actual SQL jobs before the new snapshot. let _attach = self.attach_gate.lock().await; - let previous = self - .tasks - .lock() - .expect("Worker projection task lock poisoned") - .remove(runtime_id); - if let Some(previous) = previous { - previous.abort(); - let _ = previous.await; - } + let operations = self.replace_runtime(runtime_id).await; let mut subscription = broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?; @@ -148,12 +218,25 @@ impl WorkerProjectionService { break; }; let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. }); - if let Err(error) = self.apply_broker_event(runtime_id, event) { + if let Err(error) = self + .apply_broker_event_owned_async(runtime_id, event, operations.clone()) + .await + { tracing::warn!( runtime_id, error = %error, "failed to initialize durable Worker projection; restarting from a fresh snapshot" ); + if matches!( + error, + crate::Error::DbOperationOutcomeUnknown + | crate::Error::DbExecutorUnavailable + | crate::Error::DbCapacityExceeded + ) { + // Do not automatically replay a possibly committed mutation or + // spin on executor overload; attachment must be retried explicitly. + return Err(RuntimeSubscriptionBrokerError::Closed); + } subscription.restart()?; continue; } @@ -169,12 +252,27 @@ impl WorkerProjectionService { let Some(service) = service.upgrade() else { return; }; - if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) { + if let Err(error) = service + .apply_broker_event_owned_async( + task_runtime_id.as_str(), + event, + operations.clone(), + ) + .await + { tracing::warn!( runtime_id = %task_runtime_id, error = %error, "failed to update durable Worker projection; restarting from a fresh snapshot" ); + if matches!( + error, + crate::Error::DbOperationOutcomeUnknown + | crate::Error::DbExecutorUnavailable + | crate::Error::DbCapacityExceeded + ) { + return; + } if subscription.restart().is_err() { return; } @@ -184,10 +282,61 @@ impl WorkerProjectionService { self.tasks .lock() .expect("Worker projection task lock poisoned") - .insert(runtime_id, task); + .get_mut(&runtime_id) + .expect("Runtime lifetime installed before initialization") + .consumer = Some(task); Ok(()) } + #[cfg(test)] + async fn seed_observation_async( + self: &Arc, + runtime_id: &str, + worker: &protocol::subscription::SubscriptionWorker, + observed_at: &str, + ) -> crate::Result<()> { + let service = self.clone(); + let runtime_id = runtime_id.to_string(); + let worker = worker.clone(); + let observed_at = observed_at.to_string(); + self.store + .db_executor() + .run(move || service.seed_observation(&runtime_id, &worker, &observed_at)) + .await + } + + pub async fn refresh_async(self: &Arc, worker: &RuntimeWorkerRef) -> crate::Result<()> { + let service = self.clone(); + let worker = worker.clone(); + self.store + .db_executor() + .run(move || service.refresh(&worker)) + .await + } + + async fn apply_broker_event_owned_async( + self: &Arc, + runtime_id: &str, + event: BrokerSubscriptionEvent, + operations: Arc, + ) -> crate::Result<()> { + let operation = operations + .acquire_owned() + .await + .expect("operation gate closed"); + let service = self.clone(); + let runtime_id = runtime_id.to_string(); + self.store + .db_executor() + .run(move || { + // The permit must outlive SQL commit AND ordered publication, even + // when the executor result receiver has been dropped by cancellation. + let _operation = operation; + service.apply_broker_event(&runtime_id, event) + }) + .await + } + pub fn seed_observation( &self, runtime_id: &str, @@ -313,7 +462,11 @@ mod tests { tempfile::TempDir, ) { let temp = tempfile::tempdir().unwrap(); - let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap()); + let store = Arc::new( + SqliteWorkspaceStore::open(temp.path().join("workspace.db")) + .unwrap() + .with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()), + ); store .upsert_account(&AccountRecord { account_id: "owner".to_string(), @@ -355,6 +508,266 @@ mod tests { } } + fn observation( + state: protocol::subscription::SubscriptionWorkerState, + ) -> protocol::subscription::SubscriptionWorker { + protocol::subscription::SubscriptionWorker { + restore_observation_token: None, + worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(), + runtime_id: Some("runtime-a".to_string()), + resource_key: None, + availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, + worker_state: None, + state, + has_running_internal_workers: false, + workspace_id: Some(WORKSPACE_ID.to_string()), + display_name: None, + profile: None, + job: None, + workdir_attachments: Vec::new(), + } + } + + // Real second-connection contention; the controller releases automatically on + // failure, so a regression that blocks the Tokio thread cannot hang the suite. + struct WriteLock { + release: Option>, + controller: Option>, + } + + impl WriteLock { + fn acquire(path: std::path::PathBuf) -> Self { + let (release, released) = std::sync::mpsc::channel(); + let (locked, ready) = std::sync::mpsc::channel(); + let controller = std::thread::spawn(move || { + let connection = rusqlite::Connection::open(path).unwrap(); + connection.execute_batch("BEGIN IMMEDIATE").unwrap(); + locked.send(()).unwrap(); + let _ = released.recv_timeout(std::time::Duration::from_secs(3)); + connection.execute_batch("ROLLBACK").unwrap(); + }); + ready.recv().unwrap(); + Self { + release: Some(release), + controller: Some(controller), + } + } + + fn release(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + } + } + + impl Drop for WriteLock { + fn drop(&mut self) { + self.release(); + if let Some(controller) = self.controller.take() { + controller.join().unwrap(); + } + } + } + + async fn wait_for_sql(service: &WorkerProjectionService) { + tokio::time::timeout(std::time::Duration::from_secs(1), async { + // With the second connection holding the write lock, a held ordering + // gate identifies the accepted synchronous mutation, not a fake future. + while service.ordering.try_lock().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("projection job did not enter synchronous SQL"); + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload() + { + use protocol::subscription::SubscriptionWorkerState; + let (store, service, temp) = fixture().await; + store + .upsert_worker_registry(&worker("runtime-a", "worker-a")) + .unwrap(); + let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); + assert_eq!(subscription.take_snapshot().len(), 1); + let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); + let publishing_service = service.clone(); + let publication = tokio::spawn(async move { + publishing_service + .seed_observation_async( + "runtime-a", + &observation(SubscriptionWorkerState::Idle), + "2", + ) + .await + }); + wait_for_sql(&service).await; + let started = std::time::Instant::now(); + let (control, controlled) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + control.send("control progressed").unwrap(); + }); + let (_, control_result) = tokio::join!( + tokio::time::sleep(std::time::Duration::from_millis(20)), + controlled, + ); + assert_eq!(control_result.unwrap(), "control progressed"); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "SQLite blocked the current-thread executor" + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + lock.release(); + publication.await.unwrap().unwrap(); + let committed = subscription.recv().await.unwrap(); + let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); + let snapshot = snapshot.take_snapshot(); + assert_eq!( + committed.changes, + vec![WorkerCatalogChange::Upsert(snapshot[0].clone())] + ); + assert_eq!( + snapshot[0].observation.as_ref().unwrap().worker.state, + SubscriptionWorkerState::Idle + ); + service + .seed_observation_async( + "runtime-a", + &observation(SubscriptionWorkerState::Running), + "3", + ) + .await + .unwrap(); + // Delayed readers receive the first transaction's payload, not a re-query + // of mutable authority after a later commit. + let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else { + panic!("expected upsert"); + }; + assert_eq!( + first.observation.as_ref().unwrap().worker.state, + SubscriptionWorkerState::Idle + ); + service + .refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a")) + .await + .unwrap(); + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() { + use crate::runtime_subscription::BrokerSubscriptionEvent; + use protocol::subscription::SubscriptionWorkerState; + let (store, service, temp) = fixture().await; + store + .upsert_worker_registry(&worker("runtime-a", "worker-a")) + .unwrap(); + let mut subscription = service.subscribe_ordered_async(10).await.unwrap(); + subscription.take_snapshot(); + let operations = service.replace_runtime("runtime-a").await; + let old_operations = operations.clone(); + let mut lock = WriteLock::acquire(temp.path().join("workspace.db")); + let old_service = service.clone(); + let consumer = tokio::spawn(async move { + old_service + .apply_broker_event_owned_async( + "runtime-a", + BrokerSubscriptionEvent::Event { + payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted { + worker: observation(SubscriptionWorkerState::Running), + }, + }, + operations, + ) + .await + .unwrap(); + }); + service + .tasks + .lock() + .unwrap() + .get_mut("runtime-a") + .unwrap() + .consumer = Some(consumer); + wait_for_sql(&service).await; + let replacing_service = service.clone(); + let mut replacement = tokio::spawn(async move { + let _attach = replacing_service.attach_gate.lock().await; + replacing_service.replace_runtime("runtime-a").await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) + .await + .is_err() + ); + replacement.abort(); + assert!(replacement.await.unwrap_err().is_cancelled()); + assert_eq!( + old_operations.available_permits(), + 0, + "cancelled drain lost accepted SQL ownership" + ); + // A second replacement must still observe the old gate after cancellation. + let replacing_service = service.clone(); + let mut replacement = tokio::spawn(async move { + let _attach = replacing_service.attach_gate.lock().await; + let operations = replacing_service.replace_runtime("runtime-a").await; + replacing_service + .apply_broker_event_owned_async( + "runtime-a", + BrokerSubscriptionEvent::Snapshot { + snapshot: protocol::subscription::SubscriptionSnapshot::Workers { + workers: vec![observation(SubscriptionWorkerState::Idle)], + }, + }, + operations, + ) + .await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement) + .await + .is_err() + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + lock.release(); + replacement.await.unwrap().unwrap(); + assert_eq!(old_operations.available_permits(), 1); + let old_commit = subscription.recv().await.unwrap(); + let new_commit = subscription.recv().await.unwrap(); + for (commit, expected) in [ + (old_commit, SubscriptionWorkerState::Running), + (new_commit, SubscriptionWorkerState::Idle), + ] { + let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else { + panic!("expected one committed upsert"); + }; + assert_eq!(record.observation.as_ref().unwrap().worker.state, expected); + } + let mut snapshot = service.subscribe_ordered_async(10).await.unwrap(); + assert_eq!( + snapshot.take_snapshot()[0] + .observation + .as_ref() + .unwrap() + .worker + .state, + SubscriptionWorkerState::Idle + ); + assert!(matches!( + subscription.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + service.shutdown_async().await; + assert!(service.tasks.lock().unwrap().is_empty()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn snapshot_registration_is_atomic_with_concurrent_publication() { let (store, service, _temp) = fixture().await; diff --git a/crates/workspace-server/src/worker_source.rs b/crates/workspace-server/src/worker_source.rs index 3d51476d1..3edbb5d3e 100644 --- a/crates/workspace-server/src/worker_source.rs +++ b/crates/workspace-server/src/worker_source.rs @@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError { Replay, #[error("source Worker is not a current member of this Workspace Runtime catalog")] WorkerCatalogMembership, + #[error("source proof authority is unavailable")] + Unavailable, #[error("source proof authority failed: {0}")] Authority(String), } +impl From for WorkerMutationSourceProofError { + fn from(error: crate::Error) -> Self { + match error { + crate::Error::DbCapacityExceeded + | crate::Error::DbExecutorUnavailable + | crate::Error::DbOperationOutcomeUnknown + | crate::Error::Sqlite(_) => Self::Unavailable, + other => Self::Authority(other.to_string()), + } + } +} + pub async fn verify_worker_remove_source( api: &WorkspaceApi, proof: PresentedWorkerMutationSourceProof<'_>, @@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with( let trusted = store .get_workspace_runtime_binding(&config.workspace_id, &unverified.iss) .await - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))? + .map_err(WorkerMutationSourceProofError::from)? .filter(|record| record.revoked_at.is_none()) .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?; let public_key = trusted.public_key.as_str(); @@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with( runtime_id: claims.iss.clone(), worker_id: claims.worker_id.clone(), }; - let member = store - .get_worker_registry(&config.workspace_id, &worker) - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; - if member.is_none() { - return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); - } + let workspace_id = config.workspace_id.clone(); + let catalog_store = Arc::clone(store); store - .require_current_worker_singleton_owner(&config.workspace_id, &worker) - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + .db_executor() + .run(move || { + let member = catalog_store + .get_worker_registry(&workspace_id, &worker) + .map_err(WorkerMutationSourceProofError::from)?; + if member.is_none() { + return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); + } + catalog_store + .require_current_worker_singleton_owner(&workspace_id, &worker) + .map_err(WorkerMutationSourceProofError::from)?; + Ok(()) + }) + .await?; let consumed_at = chrono::Utc::now().to_rfc3339(); let consumed = store @@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with( &consumed_at, ) .await - .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + .map_err(WorkerMutationSourceProofError::from)?; if !consumed { return Err(WorkerMutationSourceProofError::Replay); } diff --git a/crates/workspace-server/src/worker_source/runtime_request.rs b/crates/workspace-server/src/worker_source/runtime_request.rs index 9e79fd3b1..d299b520d 100644 --- a/crates/workspace-server/src/worker_source/runtime_request.rs +++ b/crates/workspace-server/src/worker_source/runtime_request.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use worker_runtime::auth::{ RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims, RuntimeRequestSourceExpectation, decode_runtime_request_source_claims, @@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError { pub(crate) exp: Option, pub(crate) now: Option, pub(crate) time_delta_seconds: Option, + /// Operational unavailability is not a proof or principal rejection. + pub(crate) availability: bool, } impl RuntimeRequestProofError { + pub fn is_availability_error(&self) -> bool { + self.availability + } + + fn unavailable(mut self, reason: &'static str) -> Self { + self.reason = reason; + self.availability = true; + self + } + fn undecoded() -> Self { Self { reason: "invalid", @@ -47,6 +61,7 @@ impl RuntimeRequestProofError { exp: None, now: None, time_delta_seconds: None, + availability: false, } } @@ -106,6 +121,18 @@ impl RuntimeRequestProofError { } } +impl From for RuntimeRequestProofError { + fn from(error: crate::Error) -> Self { + let reason = match error { + crate::Error::DbCapacityExceeded => "db_capacity_exceeded", + crate::Error::DbExecutorUnavailable => "db_executor_unavailable", + crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown", + _ => "database_authority_unavailable", + }; + Self::undecoded().unavailable(reason) + } +} + /// Domain-separated fixed-length fingerprints, not raw request-derived IDs. /// Operators can correlate a known ID by applying this same function. pub fn diagnostic_id_hash(id: &str) -> String { @@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof( body_digest: &str, ) -> Result { verify_runtime_request_source_proof_with_store( - api.store.as_ref(), + Arc::clone(&api.store), &api.config, proof, workspace_id, @@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof( #[allow(clippy::too_many_arguments)] pub async fn verify_runtime_request_source_proof_with_store( - store: &dyn ControlPlaneStore, + store: Arc, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store( #[allow(clippy::too_many_arguments)] pub(crate) async fn verify_runtime_request_source_proof_with_clock( - store: &dyn ControlPlaneStore, + store: Arc, config: &ServerConfig, proof: &str, workspace_id: &str, @@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( let unverified = decode_runtime_request_source_claims(proof) .map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?; let diagnostic = RuntimeRequestProofError::decoded(&unverified); - let audience = remote_audience(config, workspace_id) - .map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?; + let audience = remote_audience(config, workspace_id).map_err(|_| { + diagnostic + .clone() + .unavailable("backend_audience_unavailable") + })?; let trusted = store .get_workspace_runtime_binding(workspace_id, &unverified.iss) .await .map_err(|_| { diagnostic .clone() - .reason("runtime_trust_authority_unavailable") + .unavailable("runtime_trust_authority_unavailable") })? .filter(|record| record.revoked_at.is_none()) .ok_or_else(|| { @@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( &consumed_at, ) .await - .map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))? + .map_err(|_| { + diagnostic + .clone() + .unavailable("replay_authority_unavailable") + })? { return Err(diagnostic.reason("replay")); } @@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( runtime_id: claims.iss.clone(), worker_id: worker_id.to_owned(), }; - let member = store - .get_worker_registry(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_catalog_authority_unavailable") - })?; - let reserved = store - .has_active_worker_create_reservation(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_reservation_authority_unavailable") - })?; - if member.is_none() && !reserved { - return Err(diagnostic.reason("worker_catalog_membership")); - } - store - .require_current_worker_singleton_owner(workspace_id, &worker) - .map_err(|_| { - diagnostic - .clone() - .reason("worker_singleton_authority_denied") + let workspace_id = workspace_id.to_owned(); + let executor = store.db_executor(); + let stage_diagnostic = diagnostic.clone(); + // Only synchronous catalog authority lives in this closure. The trust + // and JTI async wrappers above already offload through the executor. + executor + .run(move || { + let member = store + .get_worker_registry(&workspace_id, &worker) + .map_err(|_| { + stage_diagnostic + .clone() + .unavailable("worker_catalog_authority_unavailable") + })?; + let reserved = store + .has_active_worker_create_reservation(&workspace_id, &worker) + .map_err(|_| { + stage_diagnostic + .clone() + .unavailable("worker_reservation_authority_unavailable") + })?; + if member.is_none() && !reserved { + return Err(stage_diagnostic.reason("worker_catalog_membership")); + } + store + .require_current_worker_singleton_owner(&workspace_id, &worker) + .map_err(|error| { + if matches!( + error, + crate::Error::DbCapacityExceeded + | crate::Error::DbExecutorUnavailable + | crate::Error::DbOperationOutcomeUnknown + | crate::Error::Sqlite(_) + ) { + stage_diagnostic + .clone() + .unavailable("worker_singleton_authority_unavailable") + } else { + stage_diagnostic + .clone() + .reason("worker_singleton_authority_denied") + } + })?; + Ok(()) + }) + .await + .map_err(|error: RuntimeRequestProofError| { + if error.verification == "unverified" && error.is_availability_error() { + diagnostic.clone().unavailable(error.reason) + } else { + error + } })?; } Ok(VerifiedRuntimeRequestSource { @@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock( authentication_binding_id: trusted.binding_id, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::{ + AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode, + WorkspaceRuntimeBinding, + }; + use std::time::Duration; + use worker_runtime::auth::{ + BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner, + request_body_digest, + }; + + const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa"; + const NOW: i64 = 1_700_000_000; + const PATH: &str = "/api/runtime/v1/resources/fetch"; + + // rusqlite's busy handler accepts a function pointer, not a closure. Each + // fixture owns a dedicated single-worker executor, so this thread-local + // handshake is isolated from all parallel tests and product connections. + thread_local! { + static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell, + std::sync::mpsc::Receiver<()>, + )>> = const { std::cell::RefCell::new(None) }; + } + + fn acknowledge_sqlite_contention(_attempt: i32) -> bool { + SQLITE_BUSY_HANDSHAKE.with(|slot| { + let Some((blocked, retry)) = slot.borrow_mut().take() else { + return false; + }; + let _ = blocked.send(()); + // The test releases SQLite's writer lock before allowing retry. + // Dropping the sender on panic also releases this worker promptly. + retry.recv_timeout(Duration::from_secs(3)).is_ok() + }) + } + + async fn fixture( + root: &std::path::Path, + ) -> (Arc, ServerConfig, String, String) { + let db_path = root.join("auth.db"); + let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits { + workers: std::num::NonZeroUsize::new(1).unwrap(), + queued: std::num::NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let store = Arc::new( + SqliteWorkspaceStore::open(&db_path) + .unwrap() + .with_db_executor(executor), + ); + store + .upsert_account(&AccountRecord { + account_id: "account-auth".into(), + kind: "user".into(), + handle: "auth".into(), + display_name: "Auth".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + let workspace = WorkspaceRecord { + workspace_id: WORKSPACE.into(), + owner_account_id: "account-auth".into(), + display_name: "Auth".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }; + store.upsert_workspace(&workspace).await.unwrap(); + let config = ServerConfig { + workspace_id: WORKSPACE.into(), + workspace_display_name: "Auth".into(), + workspace_created_at: "1".into(), + workspace_execution_root: root.to_owned(), + database_path: db_path, + data_root: root.to_owned(), + embedded_runtime_store_root: root.join("runtime"), + static_assets_dir: None, + auth: crate::server::AuthConfig::Passkey { + rp_id: "localhost".into(), + origin: "http://localhost".into(), + public_base_url: "http://localhost".into(), + cookie_name: "session".into(), + }, + max_records: 10, + repositories: vec![], + runtime_event_sources: vec![], + remote_runtime_sources: vec![], + backend_base_url: Some("server-auth".into()), + }; + let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap(); + store + .upsert_workspace_runtime_binding_record( + WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: "runtime-auth".into(), + display_name: "Auth".into(), + base_url: "http://127.0.0.1:1".into(), + public_key: identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer, + created_at: "1".into(), + updated_at: "1".into(), + revoked_at: None, + }, + false, + ) + .await + .unwrap(); + let binding = ControlPlaneStore::get_workspace_runtime_binding( + store.as_ref(), + WORKSPACE, + "runtime-auth", + ) + .await + .unwrap() + .unwrap(); + let proof = RuntimeRequestSourceSigner::from_identity(&identity) + .issue( + "server-auth", + WORKSPACE, + None, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + b"{}", + NOW, + 30, + ) + .unwrap(); + (store, config, proof, binding.binding_id) + } + + #[tokio::test(flavor = "current_thread")] + async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() { + let root = tempfile::tempdir().unwrap(); + let (store, config, proof, binding_id) = fixture(root.path()).await; + // A real independent SQLite writer prevents JTI insertion while trust + // reads remain possible. Rollback on Drop also releases this on panic. + let mut writer = rusqlite::Connection::open(&config.database_path).unwrap(); + let lock = writer + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .unwrap(); + let (blocked, contention_reached) = tokio::sync::oneshot::channel(); + let (retry, retry_rx) = std::sync::mpsc::channel(); + let setup_store = store.clone(); + store + .db_executor() + .run(move || -> crate::Result<()> { + SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx))); + setup_store.with_conn(|conn| { + conn.busy_handler(Some(acknowledge_sqlite_contention))?; + Ok(()) + }) + }) + .await + .unwrap(); + let request_store = store.clone(); + let request_config = config.clone(); + let request_proof = proof.clone(); + let task = tokio::spawn(async move { + verify_runtime_request_source_proof_with_clock( + request_store, + &request_config, + &request_proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(1), contention_reached) + .await + .expect( + "SQLite JTI persistence must report real writer contention without blocking Tokio", + ) + .unwrap(); + // The busy callback acknowledges an actual SQLite write blocked by the + // independent writer, then holds retry until explicitly released. + assert!( + !task.is_finished(), + "JTI must remain pending while SQLite writer is locked" + ); + let control = tokio::spawn(async { "control-responsive" }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), control) + .await + .unwrap() + .unwrap(), + "control-responsive" + ); + // This timer is the behavior being tested, not a guessed wait for SQL. + tokio::time::timeout( + Duration::from_secs(1), + tokio::time::sleep(Duration::from_millis(1)), + ) + .await + .unwrap(); + assert!( + !task.is_finished(), + "timer must advance while JTI remains blocked" + ); + lock.rollback().unwrap(); + retry.send(()).unwrap(); + let source = tokio::time::timeout(Duration::from_secs(3), task) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(source.runtime_id, "runtime-auth"); + assert_eq!(source.authentication_binding_id, binding_id); + let replay = verify_runtime_request_source_proof_with_clock( + store, + &config, + &proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ) + .await + .unwrap_err(); + assert_eq!(replay.reason, "replay"); + assert!(!replay.is_availability_error()); + } + + #[tokio::test] + async fn saturated_database_is_availability_failure_not_invalid_proof() { + let root = tempfile::tempdir().unwrap(); + let (store, config, proof, _) = fixture(root.path()).await; + // Fill the isolated executor with a blocked job and queued jobs. No + // scheduling sleeps: the worker explicitly acknowledges its start. + let executor = store.db_executor(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let blocker_executor = executor.clone(); + let blocker = tokio::spawn(async move { + blocker_executor + .run(move || -> crate::Result<()> { + let _ = started.send(()); + release_rx.recv().unwrap(); + Ok(()) + }) + .await + }); + tokio::time::timeout(Duration::from_secs(1), started_rx) + .await + .unwrap() + .unwrap(); + let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) })); + assert!(futures::poll!(queued.as_mut()).is_pending()); + let outcome = tokio::time::timeout( + Duration::from_secs(1), + verify_runtime_request_source_proof_with_clock( + store, + &config, + &proof, + WORKSPACE, + BACKEND_RESOURCE_FETCH_PERMISSION, + "POST", + PATH, + &request_body_digest(b"{}"), + &|| NOW, + ), + ) + .await; + // Release before assertions so a failed contract never strands a worker. + release.send(()).unwrap(); + blocker.await.unwrap().unwrap(); + queued.await.unwrap(); + let error = outcome + .expect("capacity rejection must not wait for queued SQL") + .unwrap_err(); + assert_eq!(error.reason, "runtime_trust_authority_unavailable"); + assert!(error.is_availability_error()); + } +} diff --git a/crates/workspace-server/src/workspace_drive_host.rs b/crates/workspace-server/src/workspace_drive_host.rs index 410c0c049..bc0bfc390 100644 --- a/crates/workspace-server/src/workspace_drive_host.rs +++ b/crates/workspace-server/src/workspace_drive_host.rs @@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost { impl WorkspaceDriveHost { pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result { let storage = storage.clone(); - let registration = tokio::task::spawn_blocking(move || Drive::register(&storage)) - .await - .map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))? - .map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?; + let registration = crate::db_executor::DbExecutor::shared() + .run(move || { + Drive::register(&storage).map_err(|error| { + Error::Store(format!("failed to register Drive storage: {error}")) + }) + }) + .await?; Ok(Self { registration, purged: AtomicBool::new(false), @@ -80,21 +83,21 @@ impl WorkspaceDriveHost { } let storage = storage.clone(); let registration = self.registration.clone(); - let drive = tokio::task::spawn_blocking(move || { - reject_symlink_ancestors(&root)?; - // Domain IMMEDIATE transactions lock the attached server authority - // as well as Drive metadata. Deletion reservation and cached-handle - // operations therefore serialize with other server DB writes. - Drive::open_with_workspace_authority( - &storage, - ®istration, - &root, - &authority_database, - ) - .map_err(Error::from) - }) - .await - .map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??; + let drive = crate::db_executor::DbExecutor::shared() + .run(move || { + reject_symlink_ancestors(&root)?; + // Domain IMMEDIATE transactions lock the attached server authority + // as well as Drive metadata. Deletion reservation and cached-handle + // operations therefore serialize with other server DB writes. + Drive::open_with_workspace_authority( + &storage, + ®istration, + &root, + &authority_database, + ) + .map_err(Error::from) + }) + .await?; *cached = Some(drive.clone()); Ok(drive) } @@ -109,9 +112,9 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root, authority_database).await?; - tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from)) + crate::db_executor::DbExecutor::shared() + .run(move || drive.fence().map_err(Error::from)) .await - .map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))? } pub(crate) async fn purge( @@ -124,20 +127,20 @@ impl WorkspaceDriveHost { return Ok(()); } let drive = self.open(storage, root.clone(), authority_database).await?; - tokio::task::spawn_blocking(move || -> Result<()> { - reject_symlink_ancestors(&root)?; - // Domain purge persists its tombstone before removing blobs/nodes/receipts. - drive.purge()?; - // Remove the empty blob root, never the data root or another Workspace. - // Missing roots are successful retries; other failures retain FeatureStorage. - match std::fs::remove_dir_all(&root) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - } - }) - .await - .map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??; + crate::db_executor::DbExecutor::shared() + .run(move || -> Result<()> { + reject_symlink_ancestors(&root)?; + // Domain purge persists its tombstone before removing blobs/nodes/receipts. + drive.purge()?; + // Remove the empty blob root, never the data root or another Workspace. + // Missing roots are successful retries; other failures retain FeatureStorage. + match std::fs::remove_dir_all(&root) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + }) + .await?; // A later signing-material/finalization failure can retry after the // feature DB has been closed and removed by FeatureStorage.delete. self.purged.store(true, Ordering::Release); diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index 8b0e0b818..eaa88d40e 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription( worker_id, runtime_id: Some(runtime_id), } => { - let connection = match WorkspaceWorker::resolve( + let connection = match WorkspaceWorker::resolve_async( &api, &runtime_id, worker_id.as_str(), - ) { + ) + .await + { Ok(worker) => worker .connect_protocol(&context) .await @@ -366,7 +368,8 @@ async fn run_workspace_workers( ) { let Ok(mut projection) = api .worker_projection - .subscribe_ordered(api.config.max_records.max(1)) + .subscribe_ordered_async(api.config.max_records.max(1)) + .await else { let _ = send_rejected( &outbound, @@ -377,11 +380,22 @@ async fn run_workspace_workers( .await; return; }; - let mut workers = projection - .take_snapshot() - .into_iter() - .filter_map(|record| project_registry_worker(&api, record)) - .collect::>(); + let Ok(mut workers) = project_registry_workers_async( + api.store.clone(), + api.config.workspace_id.clone(), + projection.take_snapshot(), + ) + .await + else { + let _ = send_rejected( + &outbound, + request_id, + SubscriptionRejectionCode::Internal, + "failed to enrich durable Worker projection".to_string(), + ) + .await; + return; + }; sort_workers(&mut workers); if send_frame( &outbound, @@ -406,7 +420,29 @@ async fn run_workspace_workers( for change in event.changes { let payload = match change { crate::worker_projection::WorkerProjectionChange::Upsert(record) => { - let Some(worker) = project_registry_worker(&api, record) else { + let workers = project_registry_workers_async( + api.store.clone(), + api.config.workspace_id.clone(), + vec![record], + ) + .await; + let Ok(mut workers) = workers else { + // Admission failure cannot silently lose a live + // update: terminate and require a fresh snapshot. + let _ = send_frame( + &outbound, + SubscriptionFrame::new(SubscriptionFramePayload::Event( + SubscriptionEvent::SubscriptionClosed { + subscription_id: subscription_id.clone(), + code: SubscriptionTerminationCode::Lagged, + message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(), + }, + )), + ) + .await; + return; + }; + let Some(worker) = workers.pop() else { continue; }; SubscriptionEventPayload::WorkerUpserted { worker } @@ -450,8 +486,28 @@ async fn run_workspace_workers( } } +async fn project_registry_workers_async( + store: std::sync::Arc, + workspace_id: String, + records: Vec, +) -> crate::Result> { + store + .db_executor() + .run(move || { + // Keep the committed Worker payload; only the existing Workspace-scoped + // workdir/history/repository enrichment reads occur inside this job. + Ok(records + .into_iter() + .filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record)) + .collect()) + }) + .await +} + +// Synchronous helpers stay synchronous; they run only inside the job above. fn project_registry_worker( - api: &WorkspaceApi, + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, record: crate::store::WorkerRegistryProjectionRecord, ) -> Option { let runtime_id = record.registry.worker.runtime_id.clone(); @@ -491,29 +547,25 @@ fn project_registry_worker( attempt_id: job.attempt_id, purpose: job.purpose, }); - if !project_working_directory(api, &record.registry.worker, &mut worker) { + if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) { return None; } Some(worker) } fn project_working_directory( - api: &WorkspaceApi, + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, worker_ref: &RuntimeWorkerRef, worker: &mut SubscriptionWorker, ) -> bool { - let Ok(links) = api - .store - .list_worker_workdir_links(&api.config.workspace_id, worker_ref) - else { + let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else { return false; }; if !links.is_empty() { let mut attachments = Vec::with_capacity(links.len()); for link in links { - let Ok(Some(workdir)) = api - .store - .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id) else { return false; }; @@ -533,9 +585,8 @@ fn project_working_directory( } worker.workdir_attachments = attachments; } else { - let Ok(has_link_history) = api - .store - .worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref) + let Ok(has_link_history) = + store.worker_workdir_link_history_exists(workspace_id, worker_ref) else { return false; }; @@ -543,13 +594,17 @@ fn project_working_directory( worker.workdir_attachments.clear(); } } - project_repository_keys(api, worker) + project_repository_keys(store, workspace_id, worker) } -fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool { +fn project_repository_keys( + store: &dyn crate::store::ControlPlaneStore, + workspace_id: &str, + worker: &mut SubscriptionWorker, +) -> bool { project_repository_keys_with(worker, |repository_id| { - api.store - .get_repository(&api.config.workspace_id, repository_id) + store + .get_repository(workspace_id, repository_id) .ok() .flatten() .map(|repository| repository.repository_key) @@ -634,9 +689,8 @@ mod tests { protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap() } - #[test] - fn external_workdir_attachment_does_not_remove_workspace_worker() { - let mut worker = SubscriptionWorker { + fn external_worker() -> SubscriptionWorker { + SubscriptionWorker { restore_observation_token: None, worker_id: worker_id(), runtime_id: Some("runtime-1".to_string()), @@ -660,8 +714,12 @@ mod tests { .unwrap(), }, ], - }; + } + } + #[test] + fn external_workdir_attachment_does_not_remove_workspace_worker() { + let mut worker = external_worker(); assert!(project_repository_keys_with(&mut worker, |_| { panic!("External Workdir must not require Repository projection") })); @@ -673,6 +731,109 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository() + { + use crate::store::{ + AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord, + WorkspaceRecord, + }; + let store = + std::sync::Arc::new(SqliteWorkspaceStore::in_memory().unwrap().with_db_executor( + crate::db_executor::DbExecutor::new(Default::default()).unwrap(), + )); + store + .upsert_account(&AccountRecord { + account_id: "owner".into(), + kind: "user".into(), + handle: "owner".into(), + display_name: "Owner".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "workspace-1".into(), + owner_account_id: "owner".into(), + display_name: "Enrichment".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .await + .unwrap(); + let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1"); + store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: "workspace-1".into(), + worker: worker_ref.clone(), + display_name: "Durable name".into(), + profile: Some("builtin:coder".into()), + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + store + .apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2") + .unwrap(); + let committed = store + .worker_registry_projection("workspace-1", &worker_ref) + .unwrap() + .unwrap(); + // Change authority after capturing the payload, as can happen while async + // enrichment is queued. It must not re-query and replace the committed Worker. + let mut later = external_worker(); + later.state = protocol::subscription::SubscriptionWorkerState::Running; + store + .apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3") + .unwrap(); + let projected = project_registry_workers_async( + store.clone(), + "workspace-1".into(), + vec![committed.clone()], + ) + .await + .unwrap(); + assert_eq!(projected.len(), 1); + let worker = &projected[0]; + assert_eq!( + worker.state, + protocol::subscription::SubscriptionWorkerState::Idle + ); + assert_eq!( + worker.availability, + protocol::subscription::SubscriptionWorkerAvailability::Observed + ); + assert_eq!(worker.display_name.as_deref(), Some("Durable name")); + assert_eq!(worker.profile.as_deref(), Some("builtin:coder")); + assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1")); + assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1")); + assert_eq!( + worker.workdir_attachments, + external_worker().workdir_attachments + ); + let mut unresolved = committed; + unresolved + .observation + .as_mut() + .unwrap() + .worker + .workdir_attachments[0] + .repository_id = Some("missing-repository".into()); + assert!( + project_registry_workers_async(store, "workspace-1".into(), vec![unresolved]) + .await + .unwrap() + .is_empty() + ); + } + async fn run_with_events(events: mpsc::Receiver) -> SubscriptionResponse { let (control_outbound, mut control_receiver) = mpsc::channel(1); let (protocol_outbound, _protocol_receiver) = mpsc::channel(1); diff --git a/docs/development/backend-db-execution.md b/docs/development/backend-db-execution.md new file mode 100644 index 000000000..dd8f1241e --- /dev/null +++ b/docs/development/backend-db-execution.md @@ -0,0 +1,95 @@ +# Backend bounded synchronous execution + +## Ownership and admission + +`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner. +Use `executor.run(move || synchronous_operation(...)).await` with owned input and +services. The closure owns the **whole** synchronous operation, including connection +acquisition, mutex wait, the existing SQL transaction, commit, and ordered local +publication. Do not split source authorization, receipts, CAS, audit, or saved +results around an await. Do not change connection ownership or introduce a read pool. + +Production stores share one DB worker and a queue of 32 jobs. Admission uses a +nonblocking queue send and a separate actual-job budget (worker count plus queue +capacity); rejection means the operation was **not accepted**. HTTP maps capacity +and executor unavailability to 503, not authentication failure or deletion conflict. +Tests can inject a dedicated executor with explicit limits, without process-global +settings or environment variables. + +Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter: + +- does not interrupt, resubmit, or retry SQL; +- does not return its queue/job slot before execution and discarded-result cleanup; +- does not release the request lifecycle lease before actual completion. + +Synchronous post-commit continuations inherit the admitted lifecycle source when +submitting another bounded job. This is not implicit inheritance by detached Tokio +tasks. An operation panic has an unknown outcome even if SQL already committed; +return a structured unknown-outcome error and diagnostic, never retry the mutation. +A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all +jobs accepted before its barrier is first polled and waits for actual cleanup; it +is not a queue close and does not include later admissions. Lifecycle callers must +stop their producers/admission before destroying resources. + +## Runtime/network boundaries + +A synchronous Runtime workflow must not monopolize the sole DB worker while +waiting for callbacks that themselves need DB authority. The Server's separate +finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job +ownership mechanism, not an unbounded `spawn_blocking` queue. + +Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can +use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an +owned workflow needs async resource locks, `run_host_workflow` polls it on a Host +worker using the original Tokio handle; do not build a replacement runtime for +cached services and do not submit to and await the Host lane from inside itself. +Control/session guards needed by an accepted mutation belong to that actual job. + +Backend Job cleanup is admitted before its durable cleanup claim. Rejection leaves +Pending/Failed ownership recoverable instead of abandoning a Running claim. Ticket +checker dispatch and timeout processing use bounded admission; overload leaves the +saved reservation/deadline diagnosable. They are not automatically retried after an +unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof +of model processing. + +Remote subscription and Workdir authorization are async callbacks: their complete +live binding check and fresh request signing are offloaded together. A queue error +must not fall back to a configured bearer token. Synchronous CLI/migration callers +and existing synchronous Runtime adapters retain their explicit blocking boundary. + +## Caller recipe (including assignment APIs) + +```rust +let store = api.store.clone(); +let workspace_id = api.workspace_id().to_owned(); +let worker = worker.clone(); +let record = api.store.db_executor().run(move || { + store.get_worker_registry(&workspace_id, &worker) +}).await?; +``` + +This is an illustration, not a new authority API. Assignment delivery scan, +reserve, prepare, acknowledgement and start authorization must keep their existing +whole transaction helpers. Additional assignment diagnostics, answer mutations, +and Ticket queries use the same owned caller boundary; do not add a second ledger +or reconstruct processing evidence from Worker liveness. + +## Regression evidence + +- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread + timer/control task progresses; accepted/cancelled queue bounds; rollback; panic + before/after commit; discarded reply cleanup and retained authority; child jobs. +- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope + opening/mutation under a separate SQLite connection's write lock; overload; + cancelled HTTP jobs and Workspace deletion drain. +- `worker_projection::tests`: real SQL contention, commit/payload publication, + ordered snapshot registration, and replacement draining a cancelled consumer's + accepted SQL before applying a new Runtime snapshot. +- Auth/proof and Runtime subscription tests protect availability classification, + replay rejection, and no bearer fallback. +- `workdir`'s `http-client` tests protect awaited fresh authorization and exact + request claims before any HTTP send. + +These fixtures use temporary authority data and observable synchronization, not +live databases or a fixed sleep to guess completion. This change does not claim a +latency benchmark or a measured production incident.