feat(backend): bound synchronous database and host execution ownership

This commit is contained in:
2026-10-12 06:43:16 +09:00
parent a6a97bca74
commit b1076f1d65
28 changed files with 9192 additions and 3901 deletions
+145 -16
View File
@@ -432,16 +432,23 @@ mod client {
use super::*;
use crate::{Workdir, WorkdirSession};
/// Provides a fresh bearer token for each Runtime request. Backend
/// implementations can mint short-lived capability tokens without making a
/// Worker-bound session expire with the token used to open it.
/// An owned, Send authorization future that can offload synchronous authority
/// work without borrowing request data or the authorization provider.
pub type WorkdirHttpAuthorizationFuture = std::pin::Pin<
Box<dyn std::future::Future<Output = Result<String, WorkdirError>> + Send + 'static>,
>;
/// Provides a fresh bearer token asynchronously for each Runtime request.
/// Backend implementations can mint short-lived capability tokens without
/// making a Worker-bound session expire with the token used to open it.
/// Blocking DB/signing work must be offloaded by the implementation.
pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync {
fn bearer_token(
&self,
method: &str,
path_and_query: &str,
body: &[u8],
) -> Result<String, WorkdirError>;
method: String,
path_and_query: String,
body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture;
}
struct FixedBearerToken(Arc<str>);
@@ -455,11 +462,12 @@ mod client {
impl WorkdirHttpAuthorization for FixedBearerToken {
fn bearer_token(
&self,
_method: &str,
_path_and_query: &str,
_body: &[u8],
) -> Result<String, WorkdirError> {
Ok(self.0.to_string())
_method: String,
_path_and_query: String,
_body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture {
let token = self.0.clone();
Box::pin(async move { Ok(token.to_string()) })
}
}
@@ -509,7 +517,9 @@ mod client {
)?;
let body = serde_json::to_vec(&request)
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
let token = authorization.bearer_token("POST", url.path(), &body)?;
let token = authorization
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
.await?;
let response = client
.post(url)
.bearer_auth(token)
@@ -560,7 +570,10 @@ mod client {
let operation = WorkdirSessionOperationRequest { operation };
let body = serde_json::to_vec(&operation)
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
let token = self.authorization.bearer_token("POST", url.path(), &body)?;
let token = self
.authorization
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
.await?;
let response = self
.client
.post(url)
@@ -864,7 +877,10 @@ mod client {
&self.base_url,
&["v1", "workdir-sessions", self.session_id.as_str()],
)?;
let token = self.authorization.bearer_token("DELETE", url.path(), &[])?;
let token = self
.authorization
.bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new())
.await?;
let response = self
.client
.delete(url)
@@ -918,11 +934,124 @@ mod client {
WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string())
}
#[cfg(test)]
mod authorization_tests {
use super::*;
use std::sync::Mutex;
#[derive(Debug, Default)]
struct RejectAuthorization {
requests: Arc<Mutex<Vec<(String, String, Vec<u8>)>>>,
}
impl WorkdirHttpAuthorization for RejectAuthorization {
fn bearer_token(
&self,
method: String,
path_and_query: String,
body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture {
let requests = self.requests.clone();
Box::pin(async move {
tokio::task::yield_now().await;
let mut requests = requests.lock().unwrap();
requests.push((method, path_and_query, body));
Err(WorkdirError::Unavailable(format!(
"authorization rejected request {}",
requests.len()
)))
})
}
}
#[tokio::test]
async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let base_url: Url = format!("http://{}", listener.local_addr().unwrap())
.parse()
.unwrap();
let client = Client::builder().no_proxy().build().unwrap();
let authorization = Arc::new(RejectAuthorization::default());
let workdir = Workdir::new("w");
let open_request = OpenWorkdirSessionRequest {
owner_worker_id: Some("worker".to_string()),
};
let error = RemoteWorkdirSession::open_with_authorization(
client.clone(),
base_url.clone(),
authorization.clone(),
workdir.id().clone(),
open_request.clone(),
)
.await
.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == "authorization rejected request 1"
));
// Seed an already-open session to exercise the public operation and
// close boundaries independently of the rejected open request.
let remote = RemoteWorkdirSession {
client,
base_url,
authorization: authorization.clone(),
workdir,
session_id: WorkdirSessionId::new("s").unwrap(),
capabilities: WorkdirSessionCapabilities::EMPTY,
closed: AtomicBool::new(false),
};
let stat = StatRequest {
path: crate::WorkdirPath::new("file").unwrap(),
};
let error = remote.stat(stat.clone()).await.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == "authorization rejected request 2"
));
let error = remote.close().await.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == "authorization rejected request 3"
));
assert_eq!(
*authorization.requests.lock().unwrap(),
vec![
(
"POST".to_string(),
"/v1/working-directories/w/sessions".to_string(),
serde_json::to_vec(&open_request).unwrap(),
),
(
"POST".to_string(),
"/v1/workdir-sessions/s/operations".to_string(),
serde_json::to_vec(&WorkdirSessionOperationRequest {
operation: WorkdirSessionOperation::Stat(stat),
})
.unwrap(),
),
(
"DELETE".to_string(),
"/v1/workdir-sessions/s".to_string(),
Vec::new(),
),
]
);
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
}
}
pub use self::RemoteWorkdirSession as ClientSession;
}
#[cfg(feature = "http-client")]
pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization};
pub use client::{
ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
};
#[cfg(test)]
mod tests {
+198 -4
View File
@@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option<String> {
})
}
pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
store: &S,
pub async fn resolve_request_actor(
store: std::sync::Arc<dyn ControlPlaneStore>,
headers: &HeaderMap,
cookie_name: &str,
) -> Result<Option<RequestActor>> {
if let Some(token) = parse_bearer(headers) {
let bearer = parse_bearer(headers);
let session = parse_cookie(headers, cookie_name);
if bearer.is_none() && session.is_none() {
return Ok(None);
}
let executor = store.db_executor();
executor
.run(move || resolve_request_actor_sync(store.as_ref(), bearer, session))
.await
}
// One owned synchronous stage includes token usage marking and the user read.
// Do not call async store wrappers here: they use the same executor themselves.
fn resolve_request_actor_sync(
store: &dyn ControlPlaneStore,
bearer: Option<String>,
session: Option<String>,
) -> Result<Option<RequestActor>> {
if let Some(token) = bearer {
let token_hash = token_hash(&token);
if let Some(api_token) = store.resolve_api_token(&token_hash)? {
if api_token
@@ -109,7 +127,7 @@ pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
}
}
if let Some(session_token) = parse_cookie(headers, cookie_name) {
if let Some(session_token) = session {
if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? {
if is_expired(&session.expires_at) {
return Ok(None);
@@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error {
#[cfg(test)]
mod tests {
use super::*;
use crate::store::{
AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord,
};
use std::sync::Arc;
fn seeded_store() -> Arc<SqliteWorkspaceStore> {
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
workers: std::num::NonZeroUsize::new(1).unwrap(),
queued: std::num::NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let store = Arc::new(
SqliteWorkspaceStore::in_memory()
.unwrap()
.with_db_executor(executor),
);
store
.upsert_account(&AccountRecord {
account_id: "account-auth".into(),
kind: "user".into(),
handle: "auth".into(),
display_name: "Auth".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.upsert_user(&UserRecord {
user_id: "user-auth".into(),
account_id: "account-auth".into(),
handle: "auth".into(),
display_name: "Auth".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.create_browser_session(&BrowserSessionRecord {
session_id: "session-auth".into(),
token_hash: token_hash("session-secret"),
user_id: "user-auth".into(),
created_at: "1".into(),
expires_at: "9999-01-01T00:00:00Z".into(),
revoked_at: None,
})
.unwrap();
store
}
#[tokio::test]
async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() {
let store = seeded_store();
let hash = token_hash("api-secret");
store
.create_api_token(&ApiTokenRecord {
token_id: "token-auth".into(),
token_hash: hash.clone(),
user_id: "user-auth".into(),
label: "Auth".into(),
created_at: "1".into(),
expires_at: None,
revoked_at: None,
last_used_at: None,
})
.unwrap();
let mut headers = HeaderMap::new();
headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap());
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
let actor = resolve_request_actor(store.clone(), &headers, "session")
.await
.unwrap()
.unwrap();
assert_eq!(actor.user_id, "user-auth");
assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken);
assert!(
store
.resolve_api_token(&hash)
.unwrap()
.unwrap()
.last_used_at
.is_some()
);
}
#[tokio::test]
async fn expired_api_token_does_not_fall_back_to_valid_browser_session() {
let store = seeded_store();
let hash = token_hash("expired-secret");
store
.create_api_token(&ApiTokenRecord {
token_id: "token-auth".into(),
token_hash: hash.clone(),
user_id: "user-auth".into(),
label: "Auth".into(),
created_at: "1".into(),
expires_at: Some("2000-01-01T00:00:00Z".into()),
revoked_at: None,
last_used_at: None,
})
.unwrap();
let mut headers = HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
"Bearer expired-secret".parse().unwrap(),
);
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
assert!(
resolve_request_actor(store.clone(), &headers, "session")
.await
.unwrap()
.is_none()
);
assert!(
store
.resolve_api_token(&hash)
.unwrap()
.unwrap()
.last_used_at
.is_none()
);
}
#[tokio::test]
async fn unknown_bearer_can_fall_back_to_valid_browser_session() {
let store = seeded_store();
let mut headers = HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
"Bearer unknown-secret".parse().unwrap(),
);
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
let actor = resolve_request_actor(store, &headers, "session")
.await
.unwrap()
.unwrap();
assert_eq!(actor.user_id, "user-auth");
assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession);
}
#[tokio::test]
async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() {
let store = seeded_store();
let executor = store.db_executor();
let (started, started_rx) = tokio::sync::oneshot::channel();
let (release, release_rx) = std::sync::mpsc::channel();
let blocker_executor = executor.clone();
let blocker = tokio::spawn(async move {
blocker_executor
.run(move || -> Result<()> {
let _ = started.send(());
release_rx.recv().unwrap();
Ok(())
})
.await
});
tokio::time::timeout(std::time::Duration::from_secs(1), started_rx)
.await
.unwrap()
.unwrap();
let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) }));
assert!(futures::poll!(queued.as_mut()).is_pending());
let mut headers = HeaderMap::new();
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
let result = tokio::time::timeout(
std::time::Duration::from_secs(1),
resolve_request_actor(store, &headers, "session"),
)
.await;
release.send(()).unwrap();
blocker.await.unwrap().unwrap();
queued.await.unwrap();
assert!(matches!(
result.expect("capacity rejection must not wait for queued SQL"),
Err(Error::DbCapacityExceeded)
));
}
#[test]
fn user_code_uses_the_uuid_random_tail() {
+957
View File
@@ -0,0 +1,957 @@
//! Bounded execution of whole synchronous DB operations, independent of Tokio.
//!
//! Accepted jobs belong to the workers, not their async waiters. Cancelling a
//! waiter does not cancel or retry SQL, release capacity, or release its lease.
use std::{
collections::BTreeSet,
future::Future,
num::NonZeroUsize,
panic::{AssertUnwindSafe, catch_unwind},
sync::{Arc, Mutex, OnceLock, mpsc},
};
use tokio::sync::{Notify, oneshot};
use crate::{Error, Result};
/// The default is one FIFO worker and at most 32 waiting operations.
#[derive(Clone, Copy, Debug)]
pub struct DbExecutorLimits {
pub workers: NonZeroUsize,
pub queued: NonZeroUsize,
}
impl Default for DbExecutorLimits {
fn default() -> Self {
Self {
workers: NonZeroUsize::new(1).unwrap(),
queued: NonZeroUsize::new(32).unwrap(),
}
}
}
/// A request can retain its lifecycle authority until actual SQL finishes.
/// Retention happens at submission, while the caller's task-local scope exists.
pub(crate) trait DbOperationLifetime: Send + Sync {
fn retain(&self) -> Box<dyn Send>;
}
tokio::task_local! {
static OPERATION_LIFETIME: Arc<dyn DbOperationLifetime>;
}
pub(crate) async fn with_operation_lifetime<F: Future>(
lifetime: Arc<dyn DbOperationLifetime>,
future: F,
) -> F::Output {
OPERATION_LIFETIME.scope(lifetime, future).await
}
#[derive(Default)]
struct Accepted {
last_job_id: u64,
unfinished: BTreeSet<u64>,
}
#[derive(Default)]
struct Tracking {
accepted: Mutex<Accepted>,
finished: Notify,
}
struct Completion {
tracking: Arc<Tracking>,
id: u64,
}
impl Drop for Completion {
fn drop(&mut self) {
self.tracking
.accepted
.lock()
.unwrap()
.unfinished
.remove(&self.id);
self.tracking.finished.notify_waiters();
}
}
struct Job {
operation: Box<dyn FnOnce() + Send>,
// Field drop order releases authority before completion becomes observable.
_lifetime: Option<Box<dyn Send>>,
lifetime_source: Option<Arc<dyn DbOperationLifetime>>,
_completion: Completion,
runtime: Option<tokio::runtime::Handle>,
diagnostics: tracing::Dispatch,
span: tracing::Span,
}
fn report_unknown(accepted_job_id: u64, phase: &'static str) {
tracing::error!(
code = "db_operation_outcome_unknown",
accepted_job_id,
phase,
error = %Error::DbOperationOutcomeUnknown,
"DB job panicked; outcome unknown, operation will not be retried"
);
}
/// Clones share the queue and actual-job budget. Workers retain only the
/// receiver, so dropping the last sender closes the queue; accepted jobs finish
/// before the dedicated threads exit. No thread is created per store open.
#[derive(Clone)]
pub struct DbExecutor {
sender: Option<mpsc::SyncSender<Job>>,
tracking: Arc<Tracking>,
max_jobs: usize,
}
impl DbExecutor {
pub fn new(limits: DbExecutorLimits) -> Result<Self> {
let max_jobs = limits
.workers
.get()
.checked_add(limits.queued.get())
.ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?;
let (sender, receiver) = mpsc::sync_channel::<Job>(limits.queued.get());
let receiver = Arc::new(Mutex::new(receiver));
for index in 0..limits.workers.get() {
let receiver = receiver.clone();
std::thread::Builder::new()
.name(format!("workspace-db-{index}"))
.spawn(move || {
loop {
let next = receiver.lock().unwrap().recv();
let Ok(job) = next else { break };
let Job {
operation,
_lifetime: lifetime,
lifetime_source,
_completion: completion,
runtime,
diagnostics,
span,
} = job;
// Keep diagnostics with their submitting subscriber as
// well as the runtime; no process-global override.
let _diagnostics = tracing::dispatcher::set_default(&diagnostics);
let _span = span.enter();
// Preserve the submitting runtime for synchronous Host
// post-commit hooks, including reply/lease destructors.
// enter() supplies context only: never drive a future
// or await/nest an executor submission in a transaction.
let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter);
// A cancelled waiter's discarded result may itself have
// a destructor. Keep both the lease and actual-job slot
// until that cleanup finishes, even if it panics.
let execute = || {
if catch_unwind(AssertUnwindSafe(operation)).is_err() {
report_unknown(completion.id, "reply_cleanup");
}
if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() {
report_unknown(completion.id, "lifetime_cleanup");
}
};
// Synchronous post-commit continuations can admit another
// bounded job. Transfer the original lifecycle source as
// well, so its authority survives the parent job/waiter.
// This does not inject context into detached Tokio tasks.
if let Some(source) = lifetime_source {
OPERATION_LIFETIME.sync_scope(source, execute);
} else {
execute();
}
drop(completion);
}
})
.map_err(|_| Error::DbExecutorUnavailable)?;
}
Ok(Self {
sender: Some(sender),
tracking: Arc::default(),
max_jobs,
})
}
pub fn shared() -> Self {
static SHARED: OnceLock<DbExecutor> = OnceLock::new();
SHARED
.get_or_init(|| {
Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self {
sender: None,
tracking: Arc::default(),
max_jobs: 0,
})
})
.clone()
}
/// Submit without waiting for queue capacity. The closure must contain the
/// entire synchronous operation/transaction, with no nested async runtime.
/// The submitting Tokio handle, when present, is entered for synchronous
/// post-commit hooks and worker cleanup; it is never used to block_on SQL.
pub async fn run<T, E>(
&self,
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
) -> std::result::Result<T, E>
where
T: Send + 'static,
E: From<Error> + Send + 'static,
{
self.submit(operation)?
.await
.unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into()))
}
/// Admit a fire-and-forget operation immediately, without spawning a Tokio
/// task or waiting for capacity. Err means the operation was not accepted;
/// Ok means the worker owns it through execution and cleanup, even after
/// the submitting task is cancelled. A panic emits a structured unknown-
/// outcome diagnostic, never an automatic retry. drain() tracks these jobs.
pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> {
self.submit(move || {
operation();
Ok::<_, Error>(())
})
.map(drop)
}
/// Shared synchronous admission: neither dropped receivers nor cancelled
/// callers remove accepted work from the queue or the actual-job budget.
fn submit<T, E>(
&self,
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
) -> std::result::Result<oneshot::Receiver<std::result::Result<T, E>>, E>
where
T: Send + 'static,
E: From<Error> + Send + 'static,
{
let sender = self
.sender
.as_ref()
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
let runtime = tokio::runtime::Handle::try_current().ok();
let diagnostics = tracing::dispatcher::get_default(Clone::clone);
let span = tracing::Span::current();
let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok();
let lifetime = lifetime_source.as_ref().map(|source| source.retain());
let (reply, result) = oneshot::channel();
// Serialize admission and acceptance snapshots. Worker completion cannot
// race acceptance, and cancelled waiters never alter this actual-job cap.
{
let mut accepted = self.tracking.accepted.lock().unwrap();
if accepted.unfinished.len() >= self.max_jobs {
return Err(Error::DbCapacityExceeded.into());
}
let id = accepted
.last_job_id
.checked_add(1)
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
// Never reuse the identity of a rejected job: its destructor may run
// after another submission has acquired the admission lock.
accepted.last_job_id = id;
let job = Job {
operation: Box::new(move || {
let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| {
report_unknown(id, "operation");
Err(Error::DbOperationOutcomeUnknown.into())
});
let _ = reply.send(outcome);
}),
_lifetime: lifetime,
lifetime_source,
runtime,
diagnostics,
span,
_completion: Completion {
tracking: self.tracking.clone(),
id,
},
};
match sender.try_send(job) {
Ok(()) => {
accepted.unfinished.insert(id);
}
Err(error) => {
// The rejected Job's completion also locks tracking; drop it only
// after releasing the admission lock.
drop(accepted);
return Err(match error {
mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded,
mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable,
}
.into());
}
}
}
Ok(result)
}
/// Wait for every job accepted before this barrier is first polled (not
/// later submissions). This bypasses queue capacity and survives cancelled
/// job waiters. It does not close the queue or stop new submissions.
pub async fn drain(&self) -> Result<()> {
if self.sender.is_none() {
return Err(Error::DbExecutorUnavailable);
}
let through = self.tracking.accepted.lock().unwrap().last_job_id;
loop {
let notified = self.tracking.finished.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self
.tracking
.accepted
.lock()
.unwrap()
.unfinished
.first()
.is_none_or(|id| *id > through)
{
return Ok(());
}
notified.await;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use rusqlite::{Connection, TransactionBehavior, ffi};
use std::{ffi::c_void, time::Duration};
fn executor(queued: usize) -> DbExecutor {
DbExecutor::new(DbExecutorLimits {
workers: NonZeroUsize::new(1).unwrap(),
queued: NonZeroUsize::new(queued).unwrap(),
})
.unwrap()
}
fn database() -> (tempfile::TempDir, Connection, Connection) {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("jobs.sqlite");
let owner = Connection::open(&path).unwrap();
owner
.execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)")
.unwrap();
let worker = Connection::open(&path).unwrap();
(dir, owner, worker)
}
struct BusyGate {
waiting: Option<oneshot::Sender<()>>,
release: mpsc::Receiver<()>,
}
// A fixture-owned callback proves SQLite actually reached its lock wait.
// Unlike busy_handler's fn pointer API, the SQLite API carries a per-call
// context, so tests need neither process-global state nor guessed sleeps.
struct BusyRegistration {
db: *mut ffi::sqlite3,
_gate: Box<BusyGate>,
}
impl BusyRegistration {
fn install(conn: &Connection, gate: BusyGate) -> Self {
unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 {
// The Box remains registered only while this connection and
// BusyRegistration are alive, on this same dedicated worker.
let gate = unsafe { &mut *context.cast::<BusyGate>() };
if let Some(waiting) = gate.waiting.take() {
let _ = waiting.send(());
if gate.release.recv().is_ok() {
return 1;
}
}
0
}
let mut gate = Box::new(gate);
// SAFETY: conn owns the handle; registration is dropped before conn.
let db = unsafe { conn.handle() };
let result = unsafe {
ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast())
};
assert_eq!(result, ffi::SQLITE_OK);
Self { db, _gate: gate }
}
}
impl Drop for BusyRegistration {
fn drop(&mut self) {
// SAFETY: unregister before releasing the callback's boxed context.
unsafe {
ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut());
}
}
}
async fn observed<T>(future: impl Future<Output = T>) -> T {
tokio::time::timeout(Duration::from_secs(5), future)
.await
.expect("DB job did not reach the expected observable state")
}
#[tokio::test(flavor = "current_thread")]
async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() {
let (_dir, mut owner, mut worker) = database();
let lock = owner
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let executor = executor(1);
let (waiting, blocked) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut write = Box::pin(executor.run(move || -> Result<()> {
let _busy = BusyRegistration::install(
&worker,
BusyGate {
waiting: Some(waiting),
release: permit,
},
);
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
Ok(())
}));
assert!(futures::poll!(&mut write).is_pending());
observed(blocked).await.unwrap();
let control = tokio::spawn(async { 42 });
assert_eq!(observed(control).await.unwrap(), 42);
// The timer itself is the contract under test, not synchronization.
observed(tokio::time::sleep(Duration::from_millis(1))).await;
assert!(futures::poll!(&mut write).is_pending());
lock.rollback().unwrap();
release.send(()).unwrap();
observed(write).await.unwrap();
executor.drain().await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
}
struct Lease(Arc<std::sync::atomic::AtomicUsize>);
struct Retained(Arc<std::sync::atomic::AtomicUsize>);
impl DbOperationLifetime for Lease {
fn retain(&self) -> Box<dyn Send> {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Box::new(Retained(self.0.clone()))
}
}
impl Drop for Retained {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() {
let (_dir, mut owner, mut worker) = database();
let lock = owner
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let executor = executor(1);
let (waiting, blocked) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut running = Box::pin(executor.run(move || -> Result<()> {
let _busy = BusyRegistration::install(
&worker,
BusyGate {
waiting: Some(waiting),
release: permit,
},
);
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
Ok(())
}));
assert!(futures::poll!(&mut running).is_pending());
observed(blocked).await.unwrap();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap();
let mut queued = Box::pin(with_operation_lifetime(
lease,
executor.run(move || -> Result<()> {
let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
// FIFO is observable in durable SQL, not inferred from waiter order.
let first: i64 =
tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| {
row.get(0)
})?;
assert_eq!(first, 1);
tx.execute("INSERT INTO writes VALUES (2)", [])?;
tx.commit()?;
Ok(())
}),
));
assert!(futures::poll!(&mut queued).is_pending());
drop(queued);
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
assert!(matches!(
executor.run(|| Ok::<_, Error>(())).await,
Err(Error::DbCapacityExceeded)
));
let mut drained = Box::pin(executor.drain());
assert!(futures::poll!(&mut drained).is_pending());
lock.rollback().unwrap();
release.send(()).unwrap();
observed(running).await.unwrap();
observed(drained).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
2
);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() {
let (dir, owner, mut parent_conn) = database();
let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap();
let parent = executor(1);
let child = executor(1);
let child_owner = child.clone();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let source: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let (entered, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let (child_entered, child_running) = oneshot::channel();
let (finish_child, child_permit) = mpsc::channel();
let mut operation = Box::pin(with_operation_lifetime(
source,
parent.run(move || -> Result<()> {
let _ = entered.send(());
let _ = permit.recv();
let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
child_owner.dispatch(move || {
let tx = child_conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let committed: i64 = tx
.query_row("SELECT count(*) FROM writes", [], |row| row.get(0))
.unwrap();
assert_eq!(committed, 1, "child must see its parent's committed SQL");
let _ = child_entered.send(());
let _ = child_permit.recv();
tx.execute("INSERT INTO writes VALUES (2)", []).unwrap();
tx.commit().unwrap();
})?;
Ok(())
}),
));
assert!(futures::poll!(&mut operation).is_pending());
observed(running).await.unwrap();
drop(operation);
release.send(()).unwrap();
observed(child_running).await.unwrap();
observed(parent.drain()).await.unwrap();
assert_eq!(
count.load(std::sync::atomic::Ordering::SeqCst),
1,
"accepted continuation lost authority when its parent finished"
);
let mut drained = Box::pin(child.drain());
assert!(futures::poll!(&mut drained).is_pending());
finish_child.send(()).unwrap();
observed(drained).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
2
);
}
#[tokio::test]
async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() {
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let error = executor
.run(move || -> Result<()> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.execute("INSERT INTO writes VALUES (NULL)", [])?;
tx.commit()?;
Ok(())
})
.await
.unwrap_err();
assert!(matches!(error, Error::Sqlite(_)));
executor.drain().await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
0
);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[tokio::test]
async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() {
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let error = executor
.run(move || -> Result<()> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
panic!("lost outcome after commit");
})
.await
.unwrap_err();
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
executor.drain().await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42);
}
#[tokio::test]
async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() {
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let error = with_operation_lifetime(
lease,
executor.run(move || -> Result<()> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
panic!("lost outcome before commit");
}),
)
.await
.unwrap_err();
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
executor.drain().await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
0
);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
struct DiscardedReplyHook {
expected_runtime: tokio::runtime::Id,
lease_count: Arc<std::sync::atomic::AtomicUsize>,
dropped: Option<oneshot::Sender<(bool, usize)>>,
}
impl Drop for DiscardedReplyHook {
fn drop(&mut self) {
let runtime =
tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context");
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
let same_runtime = runtime.id() == self.expected_runtime;
let dropped = self.dropped.take().unwrap();
tokio::spawn(async move {
let _ = dropped.send((same_runtime, retained));
});
}
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop()
{
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let (started, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
let _ = started.send(());
let _ = permit.recv();
Ok(())
}));
assert!(futures::poll!(&mut blocker).is_pending());
observed(running).await.unwrap();
let expected_runtime = tokio::runtime::Handle::current().id();
let (hook, published) = oneshot::channel();
let (finish, finishing) = mpsc::channel();
let (dropped, reply_dropped) = oneshot::channel();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let operation_count = count.clone();
let mut queued = Box::pin(with_operation_lifetime(
lease,
executor.run(move || -> Result<_> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
// Model synchronous Host hooks: publish only after commit, checking
// try_current and spawning without driving any future on this worker.
let runtime = tokio::runtime::Handle::try_current()
.expect("post-commit hook lost runtime context");
assert_eq!(runtime.id(), expected_runtime);
tokio::spawn(async move {
let _ = hook.send(());
});
let _ = finishing.recv();
Ok(DiscardedReplyHook {
expected_runtime,
lease_count: operation_count,
dropped: Some(dropped),
})
}),
));
assert!(futures::poll!(&mut queued).is_pending());
drop(queued);
release.send(()).unwrap();
observed(blocker).await.unwrap();
observed(published).await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
let mut drain = Box::pin(executor.drain());
assert!(futures::poll!(&mut drain).is_pending());
finish.send(()).unwrap();
assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1));
observed(drain).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() {
let executor = DbExecutor::new(DbExecutorLimits {
workers: NonZeroUsize::new(2).unwrap(),
queued: NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let mut releases = Vec::new();
// Observe each worker executing before submitting the next operation,
// so queue occupancy and overload are deterministic, not timing guesses.
for _ in 0..2 {
let (started, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut operation = Box::pin(executor.run(move || -> Result<()> {
let _ = started.send(());
let _ = permit.recv();
Ok(())
}));
assert!(futures::poll!(&mut operation).is_pending());
observed(running).await.unwrap();
drop(operation);
releases.push(release);
}
let (finished, mut queued_finished) = oneshot::channel();
let mut queued = Box::pin(executor.run(move || -> Result<()> {
let _ = finished.send(());
Ok(())
}));
assert!(futures::poll!(&mut queued).is_pending());
drop(queued);
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
for _ in 0..8 {
let rejected_runs = rejected_runs.clone();
assert!(matches!(
executor
.run(move || {
rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok::<_, Error>(())
})
.await,
Err(Error::DbCapacityExceeded)
));
}
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
assert!(matches!(
queued_finished.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
let mut drain = Box::pin(executor.drain());
assert!(futures::poll!(&mut drain).is_pending());
for release in releases {
release.send(()).unwrap();
}
observed(drain).await.unwrap();
observed(queued_finished).await.unwrap();
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[derive(Default)]
struct UnknownDiagnostic {
code: Option<String>,
phase: Option<String>,
accepted_job_id: Option<u64>,
}
impl tracing::field::Visit for UnknownDiagnostic {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
match field.name() {
"code" => self.code = Some(value.to_owned()),
"phase" => self.phase = Some(value.to_owned()),
_ => {}
}
}
fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
if field.name() == "accepted_job_id" {
self.accepted_job_id = Some(value);
}
}
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
}
#[derive(Clone, Default)]
struct DiagnosticCapture(Arc<Mutex<Vec<UnknownDiagnostic>>>);
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for DiagnosticCapture {
fn on_event(
&self,
event: &tracing::Event<'_>,
_: tracing_subscriber::layer::Context<'_, S>,
) {
let mut diagnostic = UnknownDiagnostic::default();
event.record(&mut diagnostic);
self.0.lock().unwrap().push(diagnostic);
}
}
struct DispatchCleanup {
expected_runtime: tokio::runtime::Id,
lease_count: Arc<std::sync::atomic::AtomicUsize>,
started: Option<oneshot::Sender<(bool, usize)>>,
release: mpsc::Receiver<()>,
}
impl Drop for DispatchCleanup {
fn drop(&mut self) {
// No panicking assertions during unwind: report observations to the
// test, and gate cleanup so drain/lifetime behavior is observable.
let same_runtime = tokio::runtime::Handle::try_current()
.is_ok_and(|runtime| runtime.id() == self.expected_runtime);
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
if let Some(started) = self.started.take() {
let _ = started.send((same_runtime, retained));
}
let _ = self.release.recv();
}
}
#[tokio::test(flavor = "current_thread")]
async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry()
{
use tracing_subscriber::layer::SubscriberExt;
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let (started, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
let _ = started.send(());
let _ = permit.recv();
Ok(())
}));
assert!(futures::poll!(&mut blocker).is_pending());
observed(running).await.unwrap();
drop(blocker); // Cancellation does not free its running slot.
let expected_runtime = tokio::runtime::Handle::current().id();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let operation_count = count.clone();
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let operation_attempts = attempts.clone();
let (cleanup_started, cleaning_up) = oneshot::channel();
let (finish, finishing) = mpsc::channel();
// Model multiple tracing contexts without setting a global subscriber.
// This also prevents tracing's single-subscriber callsite optimization
// from caching an unsubscribed parallel test's Interest::never here.
let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry());
let captured = DiagnosticCapture::default();
let diagnostics =
tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone()));
with_operation_lifetime(lease, async {
tracing::dispatcher::with_default(&diagnostics, || {
executor.dispatch(move || {
let _cleanup = DispatchCleanup {
expected_runtime,
lease_count: operation_count,
started: Some(cleanup_started),
release: finishing,
};
operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let tx = worker
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
tx.execute("INSERT INTO writes VALUES (1)", []).unwrap();
tx.commit().unwrap();
panic!("dispatched post-commit outcome lost");
})
})
})
.await
.unwrap(); // Dispatch is admitted synchronously; no waiter survives.
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let rejected_operation = rejected_runs.clone();
assert!(matches!(
executor.dispatch(move || {
rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}),
Err(Error::DbCapacityExceeded)
));
release.send(()).unwrap();
assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1));
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
let mut drain = Box::pin(executor.drain());
assert!(futures::poll!(&mut drain).is_pending());
finish.send(()).unwrap();
observed(drain).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
{
let events = captured.0.lock().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(
events[0].code.as_deref(),
Some("db_operation_outcome_unknown")
);
assert_eq!(events[0].phase.as_deref(), Some("operation"));
assert!(events[0].accepted_job_id.is_some());
}
let (completed, completion) = oneshot::channel();
executor
.dispatch(move || {
let _ = completed.send(());
})
.unwrap();
observed(executor.drain()).await.unwrap();
observed(completion).await.unwrap();
}
}
@@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
migrate_content_state_v85_to_v86(&conn).unwrap();
drop(conn);
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
};
let job = store
@@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
migrate_content_state_v85_to_v86(&conn).unwrap();
drop(conn);
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
};
let job = store
@@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
};
let migrated = store
@@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
});
let archived = store
@@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
});
let service =
@@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() {
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
});
let service =
@@ -533,6 +533,7 @@ mod existing_workspace_provisioning {
// This fixture contains only the schema-87 cutover authorities, not the
// unrelated full Server schema. Exercise the real service/store methods.
Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
})
}
@@ -644,6 +645,7 @@ mod existing_workspace_provisioning {
drop(service);
drop(store);
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())),
});
let service = WorkspaceSigningIdentityService::new(
+126 -19
View File
@@ -23,7 +23,10 @@ use std::{
};
use workdir::{
Workdir, WorkdirError, WorkdirSessionHandle,
http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization},
http::{
OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization,
WorkdirHttpAuthorizationFuture,
},
};
use worker_runtime::RuntimeWorkspaceScope;
use worker_runtime::catalog::{
@@ -3473,6 +3476,38 @@ impl WorkspaceRuntimeAuthorization {
}
}
/// Offload the complete live-binding check and request signing operation.
/// Claims are minted inside the bounded job, not cached before admission.
pub(crate) async fn issue_async(
&self,
method: String,
path_and_query: String,
operation: String,
worker_id: Option<String>,
body: Vec<u8>,
) -> Result<String, RuntimeDiagnostic> {
let executor = self.store.db_executor();
let authorization = self.clone();
executor
.run(move || {
Ok::<_, Error>(authorization.issue(
&method,
&path_and_query,
&operation,
worker_id.as_deref(),
&body,
))
})
.await
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
HostDiagnosticSeverity::Error,
error.to_string(),
)
})?
}
pub(crate) fn issue(
&self,
method: &str,
@@ -3645,25 +3680,24 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization {
impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization {
fn bearer_token(
&self,
method: &str,
path_and_query: &str,
body: &[u8],
) -> Result<String, WorkdirError> {
if let Some(authorization) = &self.workspace_authorization {
return authorization
.issue(
method,
path_and_query,
workspace_runtime_operation(method, path_and_query),
None,
body,
method: String,
path_and_query: String,
body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture {
let authorization = self.clone();
Box::pin(async move {
if let Some(authorizer) = &authorization.workspace_authorization {
let operation = workspace_runtime_operation(&method, &path_and_query).to_string();
return authorizer
.issue_async(method, path_and_query, operation, None, body)
.await
.map_err(|error| WorkdirError::Unavailable(error.message));
}
authorization.fallback_bearer_token.ok_or_else(|| {
WorkdirError::Unavailable(
"remote Runtime does not have bearer authorization configured".to_string(),
)
.map_err(|error| WorkdirError::Unavailable(error.message));
}
self.fallback_bearer_token.clone().ok_or_else(|| {
WorkdirError::Unavailable(
"remote Runtime does not have bearer authorization configured".to_string(),
)
})
})
}
}
@@ -6344,6 +6378,79 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::thread;
#[tokio::test]
async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() {
use crate::db_executor::{DbExecutor, DbExecutorLimits};
use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding};
use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore;
let executor = DbExecutor::new(DbExecutorLimits {
workers: std::num::NonZeroUsize::new(1).unwrap(),
queued: std::num::NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let store = Arc::new(
SqliteWorkspaceStore::in_memory()
.unwrap()
.with_db_executor(executor.clone()),
);
let service = WorkspaceSigningIdentityService::new(
store.clone(),
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
);
let authorization = RemoteWorkdirAuthorization {
workspace_authorization: Some(WorkspaceRuntimeAuthorization::new(
store,
service,
"https://backend.test",
WorkspaceRuntimeBinding {
workspace_id: "workspace".into(),
runtime_id: "runtime".into(),
display_name: "Runtime".into(),
base_url: "https://runtime.test".into(),
public_key_fingerprint: "unused".into(),
public_key: "unused".into(),
binding_id: "binding".into(),
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
created_at: "1".into(),
updated_at: "1".into(),
revoked_at: None,
},
)),
fallback_bearer_token: Some("must-not-be-used".into()),
};
// Hold the running slot and fill the only queue slot. The callback must
// reject admission, not run inline or silently switch authentication.
let (release, wait) = std::sync::mpsc::channel::<()>();
let (started, running) = tokio::sync::oneshot::channel();
executor
.dispatch(move || {
let _ = started.send(());
let _ = wait.recv_timeout(Duration::from_secs(5));
})
.unwrap();
tokio::time::timeout(Duration::from_secs(5), running)
.await
.unwrap()
.unwrap();
executor.dispatch(|| {}).unwrap();
let error = authorization
.bearer_token(
"POST".into(),
"/v1/workdir-sessions/s/operations".into(),
b"{}".to_vec(),
)
.await
.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string()
));
drop(release);
executor.drain().await.unwrap();
}
#[test]
fn ticket_worker_spawn_intent_uses_worker_wire_role() {
let intent = WorkerSpawnIntent::TicketRole {
+7
View File
@@ -10,6 +10,7 @@ pub mod backend_job;
pub mod companion;
pub mod config;
pub mod config_source;
pub mod db_executor;
pub mod feature_storage;
pub mod hosts;
pub mod memory_backend;
@@ -159,6 +160,12 @@ pub enum Error {
WorkspaceSigningIdentity { code: String, message: String },
#[error("Workspace Drive error: {0}")]
Drive(#[from] workspace_drive::Error),
#[error("DB execution capacity exceeded")]
DbCapacityExceeded,
#[error("DB executor unavailable")]
DbExecutorUnavailable,
#[error("DB operation outcome unknown; do not automatically retry")]
DbOperationOutcomeUnknown,
#[error("store error: {0}")]
Store(String),
}
@@ -965,7 +965,7 @@ async fn connect_runtime(
let mut request = endpoint
.into_client_request()
.map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?;
if let Some(token) = runtime_token(config, workspace_id)? {
if let Some(token) = runtime_token(config, workspace_id).await? {
request.headers_mut().insert(
"authorization",
format!("Bearer {token}")
@@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String {
format!("{base}/v1/protocol/ws")
}
}
fn runtime_token(
async fn runtime_token(
config: &RemoteRuntimeConfig,
_workspace_id: &str,
) -> Result<Option<String>, String> {
if let Some(authorization) = config.workspace_authorization.as_ref() {
return authorization
.issue("GET", "/v1/protocol/ws", "workers:list", None, &[])
.issue_async(
"GET".to_string(),
"/v1/protocol/ws".to_string(),
"workers:list".to_string(),
None,
Vec::new(),
)
.await
.map(Some)
.map_err(|error| error.message);
}
File diff suppressed because it is too large Load Diff
@@ -65,70 +65,83 @@ pub(super) fn require_candidate_grant(
/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface
/// background work; the new Job regenerates it and does not infer surface success.
pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult<bool> {
let job_api = api.clone();
let subject_id = subject_id.to_owned();
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
let Some(owner) = api
.store
.current_worker_singleton_owner(api.workspace_id(), &key)?
else {
return Ok(true);
let (worker, complete) = host_io_executor()?
.run(move || {
let api = &job_api;
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
let Some(owner) = api
.store
.current_worker_singleton_owner(api.workspace_id(), &key)?
else {
return Ok((None, true));
};
let mut summary = api
.runtime
.worker(&owner.worker)
.map_err(|e| e.into_error())?;
summary.singleton_key = Some(owner.key.clone());
require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?;
let stopped = summary.state.eq_ignore_ascii_case("stopped");
let idle = summary.state.eq_ignore_ascii_case("idle")
&& summary
.worker_state
.as_ref()
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
if !stopped && !idle {
return Ok((None, false));
}
if api
.store
.current_worker_singleton_owner(api.workspace_id(), &key)?
.as_ref()
.map(|o| &o.worker)
!= Some(&owner.worker)
{
return Err(Error::RepositoryConflict(
"legacy consolidation owner moved during cutover".into(),
)
.into());
}
if !summary.state.eq_ignore_ascii_case("stopped") {
let stopped = api
.runtime
.stop_worker(
&owner.worker,
WorkerLifecycleRequest {
reason: Some(
"Retire legacy subjektiv consolidator after foreground completion"
.into(),
),
ticket_assignment: None,
},
)
.map_err(|e| e.into_error())?;
if stopped.state != InternalWorkerOperationState::Accepted {
return Ok((None, false));
}
}
if !api
.runtime
.worker(&owner.worker)
.map_err(|e| e.into_error())?
.state
.eq_ignore_ascii_case("stopped")
{
return Ok((None, false));
}
Ok::<_, ApiError>((Some(owner.worker), false))
})
.await?;
let Some(worker) = worker else {
return Ok(complete);
};
let mut summary = api
.runtime
.worker(&owner.worker)
.map_err(|e| e.into_error())?;
summary.singleton_key = Some(owner.key.clone());
require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?;
let stopped = summary.state.eq_ignore_ascii_case("stopped");
let idle = summary.state.eq_ignore_ascii_case("idle")
&& summary
.worker_state
.as_ref()
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
if !stopped && !idle {
return Ok(false);
}
if api
.store
.current_worker_singleton_owner(api.workspace_id(), &key)?
.as_ref()
.map(|o| &o.worker)
!= Some(&owner.worker)
{
return Err(Error::RepositoryConflict(
"legacy consolidation owner moved during cutover".into(),
)
.into());
}
if !summary.state.eq_ignore_ascii_case("stopped") {
let stopped = api
.runtime
.stop_worker(
&owner.worker,
WorkerLifecycleRequest {
reason: Some(
"Retire legacy subjektiv consolidator after foreground completion".into(),
),
ticket_assignment: None,
},
)
.map_err(|e| e.into_error())?;
if stopped.state != InternalWorkerOperationState::Accepted {
return Ok(false);
}
}
if !api
.runtime
.worker(&owner.worker)
.map_err(|e| e.into_error())?
.state
.eq_ignore_ascii_case("stopped")
{
return Ok(false);
}
let removal = WorkerRemovalService::new(api)
.execute_target_removal(
api.runtime.as_ref(),
&owner.worker,
&worker,
"subjektiv consolidation Job cutover",
None,
)
@@ -141,10 +154,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
))
.into());
}
Ok(api
.store
.current_worker_singleton_owner(api.workspace_id(), &key)?
.is_none())
let store = api.store.clone();
let workspace_id = api.workspace_id().to_owned();
api.store
.db_executor()
.run(move || {
store
.current_worker_singleton_owner(&workspace_id, &key)
.map(|owner| owner.is_none())
.map_err(ApiError::from)
})
.await
}
fn bounded_candidate_batch(
@@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m
&error,
)
});
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(
to_bytes(response.into_body(), usize::MAX)
.await
@@ -767,6 +767,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) {
std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 })
);
assert!(execution.protocol_methods().is_empty());
// SQLite jobs run on external threads; paused Tokio time cannot observe their
// progress and may auto-advance a reconnect timeout before SQL completes.
tokio::time::resume();
assert_same_worker_reconnect_and_stop(&worker, &context).await;
}
@@ -844,7 +847,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub
}
}
#[tokio::test(start_paused = true)]
#[tokio::test]
async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress()
{
let workspace = tempfile::tempdir().unwrap();
@@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
let f = Fixture::new().await;
let mut a = f.attach("1");
let (first, reply) = f.start_read("1", &mut a).await;
let mut second = Box::pin(f.app.clone().oneshot(f.operation("1")));
// Poll through dispatch/auth until the handler waits on its resource lock.
assert!(futures::poll!(&mut second).is_pending());
let request = f.operation("1");
let proof = request
.headers()
.get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER)
.unwrap()
.to_str()
.unwrap();
let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof)
.unwrap()
.jti;
let mut second = Box::pin(f.app.clone().oneshot(request));
// DB auth is now genuinely asynchronous. Observe durable proof consumption,
// not a guessed number of polls, before advancing the injected proof clock.
let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
tokio::time::timeout(BOUND, async {
loop {
assert!(futures::poll!(&mut second).is_pending());
let consumed: bool = connection
.query_row(
"SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)",
[&jti],
|row| row.get(0),
)
.unwrap();
if consumed {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("second request never authenticated before resource wait");
assert!(
a.try_recv().is_err(),
"same session read escaped resource exclusion"
@@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
assert_json(first.await.unwrap(), StatusCode::OK).await;
assert!(futures::poll!(&mut second).is_pending());
let Some(ExternalProviderCommand::Operation { response, .. }) =
tokio::time::timeout(BOUND, a.recv()).await.unwrap()
tokio::time::timeout(BOUND, async {
tokio::select! {
command = a.recv() => command,
result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()),
}
}).await.unwrap()
else {
panic!("expected second read")
};
@@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits(
}),
);
let mut deletion = Box::pin(f.app.clone().oneshot(request));
assert!(futures::poll!(&mut deletion).is_pending());
let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await;
assert!(
gate.admit(false).is_none(),
"deletion start did not fence admission"
);
tokio::time::timeout(BOUND, async {
loop {
assert!(futures::poll!(&mut deletion).is_pending());
if gate.admit(false).is_none() {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("deletion start did not fence admission after async authentication");
let session_id = session_store::new_session_id().to_string();
let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions");
let callback = f.request(
@@ -57,6 +57,19 @@ impl WorkerOperationContext {
}
Ok(())
}
pub(super) async fn from_request_async(
api: &WorkspaceApi,
context: &server_api::ServerRequestContext,
) -> std::result::Result<Self, server_api::RepositoryApiError> {
let owned_api = api.clone();
let context = context.clone();
api.store
.db_executor()
.run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context)))
.await
.map_err(|error| ApiError::from(error).into_repository_api_error())?
}
pub(super) fn from_request(
api: &WorkspaceApi,
context: &server_api::ServerRequestContext,
@@ -162,8 +175,20 @@ impl<Services> WorkspaceWorker<Services> {
source: crate::worker_source::VerifiedWorkerMutationSource,
reason: &str,
) -> std::result::Result<worker::WorkspaceResponse, String> {
self.current_record()
.map_err(|error| error.error.to_string())?;
let removal = self.removal.clone();
let identity = self.identity.clone();
self.removal
.store
.db_executor()
.run(move || {
removal
.store
.get_worker_registry(&removal.workspace_id, &identity)?
.ok_or_else(|| Error::UnknownWorker { worker: identity })
.map(drop)
})
.await
.map_err(|error| error.to_string())?;
self.removal
.execute_async(
source,
@@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor
}
impl WorkspaceWorker {
pub(crate) async fn resolve_async(
api: &WorkspaceApi,
runtime_id: &str,
reference: &str,
) -> Result<Self> {
let owned_api = api.clone();
let runtime_id = runtime_id.to_owned();
let reference = reference.to_owned();
api.store
.db_executor()
.run(move || Self::resolve(&owned_api, &runtime_id, &reference))
.await
}
pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result<Self> {
Self::resolve_with_services(
api.clone(),
@@ -239,7 +278,36 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
permission: &str,
) -> ApiResult<Option<tokio::sync::OwnedMutexGuard<()>>> {
self.current_record()?;
let worker = self.clone();
let initial_context = context.clone();
let permission = permission.to_owned();
let initial_permission = permission.clone();
let grant = self
.api
.store
.db_executor()
.run(move || -> ApiResult<Option<WorkerControlGrantRecord>> {
worker.current_record()?;
let WorkerOperationContext::WorkerControl { controller } = initial_context else {
return Ok(None);
};
if worker
.api
.store
.get_worker_registry(worker.api.workspace_id(), &controller)?
.is_none()
{
return Err(Error::UnknownWorker { worker: controller }.into());
}
Ok(Some(authorize_known_worker_permission(
&worker.api,
worker.api.workspace_id(),
&controller,
&worker.identity,
&initial_permission,
)?))
})
.await?;
match context {
#[cfg(test)]
WorkerOperationContext::Backend => Ok(None),
@@ -248,42 +316,35 @@ impl WorkspaceWorker {
Ok(None)
}
WorkerOperationContext::WorkerControl { controller } => {
if self
.api
.store
.get_worker_registry(self.api.workspace_id(), controller)?
.is_none()
{
return Err(Error::UnknownWorker {
worker: controller.clone(),
}
.into());
}
let grant = authorize_known_worker_permission(
&self.api,
self.api.workspace_id(),
controller,
&self.identity,
permission,
)?;
let grant = grant.expect("Worker control context resolved a grant");
// Await admission outside the DB lane; the recheck owns the guard
// until it finishes, even if its async waiter is cancelled.
let guard = worker_control_lock(&self.api, &grant.grant_id)
.lock_owned()
.await;
self.current_record()?;
let current_grant = authorize_known_worker_permission(
&self.api,
self.api.workspace_id(),
controller,
&self.identity,
permission,
)?;
if current_grant.grant_id != grant.grant_id {
return Err(Error::RepositoryConflict(
"Worker control grant changed while waiting for admission".into(),
)
.into());
}
Ok(Some(guard))
let worker = self.clone();
let controller = controller.clone();
self.api
.store
.db_executor()
.run(move || -> ApiResult<_> {
worker.current_record()?;
let current_grant = authorize_known_worker_permission(
&worker.api,
worker.api.workspace_id(),
&controller,
&worker.identity,
&permission,
)?;
if current_grant.grant_id != grant.grant_id {
return Err(Error::RepositoryConflict(
"Worker control grant changed while waiting for admission".into(),
)
.into());
}
Ok(Some(guard))
})
.await
}
}
}
@@ -293,8 +354,14 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
request: server_api::WorkerRestoreRequest,
) -> ApiResult<InternalWorkerRestoreResult> {
let _guard = self.authorize_operation(context, "restore").await?;
self.restore_internal(request)
let guard = self.authorize_operation(context, "restore").await?;
let worker = self.clone();
host_io_executor()?
.run(move || {
let _guard = guard;
worker.restore_internal(request)
})
.await
}
/// Trusted Backend callers still supply an explicit observed intent. Runtime
@@ -312,27 +379,43 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
request: WorkerLifecycleRequest,
) -> ApiResult<WorkerLifecycleResult> {
let _guard = self.authorize_operation(context, "stop").await?;
let result = self
.api
.runtime
.stop_worker(&self.identity, request)
.map_err(|e| e.into_error())?;
let guard = self.authorize_operation(context, "stop").await?;
let worker = self.clone();
let (result, guard) = host_io_executor()?
.run(move || -> ApiResult<_> {
let result = worker
.api
.runtime
.stop_worker(&worker.identity, request)
.map_err(|e| e.into_error())?;
Ok((result, guard))
})
.await?;
if result.state != InternalWorkerOperationState::Accepted {
return Ok(result);
}
parse_runtime_worker_id_for_registry(&self.identity.worker_id)?;
let session_lock = current_worker_session_lock(&self.api, &self.identity);
let _session_guard = session_lock.lock().await;
let session_guard = session_lock.lock_owned().await;
close_current_worker_session_locked(&self.api, &self.identity).await?;
if let Some(record) = self
.api
.store
.get_worker_registry(self.api.workspace_id(), &self.identity)?
{
sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?;
}
Ok(result)
let worker = self.clone();
host_io_executor()?
.run(move || -> ApiResult<_> {
let (_control, _session) = (guard, session_guard);
if let Some(record) = worker
.api
.store
.get_worker_registry(worker.api.workspace_id(), &worker.identity)?
{
sync_linked_workdir_after_worker_stop(
&worker.api,
&worker.identity.runtime_id,
&record,
)?;
}
Ok(result)
})
.await
}
pub(super) async fn cancel(
@@ -340,12 +423,18 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
request: WorkerLifecycleRequest,
) -> ApiResult<WorkerLifecycleResult> {
let _guard = self.authorize_operation(context, "cancel").await?;
Ok(self
.api
.runtime
.cancel_worker(&self.identity, request)
.map_err(|e| e.into_error())?)
let guard = self.authorize_operation(context, "cancel").await?;
let worker = self.clone();
host_io_executor()?
.run(move || -> ApiResult<_> {
let _guard = guard;
Ok(worker
.api
.runtime
.cancel_worker(&worker.identity, request)
.map_err(|e| e.into_error())?)
})
.await
}
pub(super) async fn input(
@@ -358,46 +447,59 @@ impl WorkspaceWorker {
} else {
"send_input"
};
let _guard = self.authorize_operation(context, permission).await?;
let guard = self.authorize_operation(context, permission).await?;
let session_lock = current_worker_session_lock(&self.api, &self.identity);
let _session_guard = session_lock.lock().await;
self.ensure_interactive()?;
let session_guard = session_lock.lock_owned().await;
self.ensure_interactive().await?;
if matches!(context, WorkerOperationContext::WorkerControl { .. })
&& request.kind == WorkerInputKind::User
{
request.kind = WorkerInputKind::UserIfIdle;
}
Ok(self
.api
.runtime
.send_input(&self.identity, request)
.map_err(|e| e.into_error())?)
let worker = self.clone();
host_io_executor()?
.run(move || -> ApiResult<_> {
let (_control, _session) = (guard, session_guard);
Ok(worker
.api
.runtime
.send_input(&worker.identity, request)
.map_err(|e| e.into_error())?)
})
.await
}
fn ensure_interactive(&self) -> ApiResult<()> {
if let Some(claim) = self
.api
async fn ensure_interactive(&self) -> ApiResult<()> {
let worker = self.clone();
self.api
.store
.get_pending_ticket_assignment_operation_for_worker(
self.api.workspace_id(),
&self.identity,
)?
{
return Err(Error::TicketAssignmentConflict(format!(
"Worker has pending Ticket claim {}; reconcile that exact operation before admitting input",
claim.operation_id,
)).into());
}
if self
.api
.store
.worker_registry_projection(self.api.workspace_id(), &self.identity)?
.and_then(|projection| projection.job)
.is_some()
{
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
}
Ok(())
.db_executor()
.run(move || -> ApiResult<()> {
if let Some(claim) = worker
.api
.store
.get_pending_ticket_assignment_operation_for_worker(
worker.api.workspace_id(),
&worker.identity,
)?
{
return Err(Error::TicketAssignmentConflict(format!(
"Worker has pending Ticket claim {}; reconcile that exact operation before admitting input",
claim.operation_id,
)).into());
}
if worker
.api
.store
.worker_registry_projection(worker.api.workspace_id(), &worker.identity)?
.and_then(|projection| projection.job)
.is_some()
{
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
}
Ok(())
})
.await
}
pub(super) async fn set_pinned(
@@ -405,33 +507,44 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
pinned: bool,
) -> ApiResult<WorkerRetentionResponse> {
let _guard = self.authorize_operation(context, "pin").await?;
let retention_state = if pinned { "pinned" } else { "normal" };
let changed = self.api.worker_projection.publish_ordered(|| {
self.api
.store
.update_worker_retention(
self.api.workspace_id(),
&self.identity,
retention_state,
now_registry_timestamp().as_str(),
)
.map(|commit| (!commit.changes.is_empty(), commit))
})?;
if !changed {
return Err(cleanup_api_error(
&self.identity.runtime_id,
"workspace_worker_retention_unknown_worker",
"Worker is not known to the Backend registry",
));
}
Ok(WorkerRetentionResponse {
workspace_id: self.api.workspace_id().to_owned(),
runtime_id: self.identity.runtime_id.clone(),
worker_id: self.identity.worker_id.clone(),
pinned,
retention_state: retention_state.into(),
})
let guard = self.authorize_operation(context, "pin").await?;
let worker = self.clone();
self.api
.store
.db_executor()
.run(move || -> ApiResult<WorkerRetentionResponse> {
// Publication and its DB mutation are one owned stage. Accepted
// work retains control admission through commit and publication.
let _guard = guard;
let retention_state = if pinned { "pinned" } else { "normal" };
let changed = worker.api.worker_projection.publish_ordered(|| {
worker
.api
.store
.update_worker_retention(
worker.api.workspace_id(),
&worker.identity,
retention_state,
now_registry_timestamp().as_str(),
)
.map(|commit| (!commit.changes.is_empty(), commit))
})?;
if !changed {
return Err(cleanup_api_error(
&worker.identity.runtime_id,
"workspace_worker_retention_unknown_worker",
"Worker is not known to the Backend registry",
));
}
Ok(WorkerRetentionResponse {
workspace_id: worker.api.workspace_id().to_owned(),
runtime_id: worker.identity.runtime_id.clone(),
worker_id: worker.identity.worker_id.clone(),
pinned,
retention_state: retention_state.into(),
})
})
.await
}
/// The caller owns the full cleanup-plan digest check. The removal service
@@ -611,7 +724,7 @@ impl WorkspaceWorkerMethodSender {
.await?;
let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity);
let _session_guard = session_lock.lock().await;
self.worker.ensure_interactive()?;
self.worker.ensure_interactive().await?;
let WorkerOperationContext::Browser { source, .. } = context else {
return Err(Error::WorkspacePermissionDenied(
"Browser Worker protocol context required".into(),
@@ -72,6 +72,28 @@ pub(super) struct RequestLease {
callback: bool,
}
impl crate::db_executor::DbOperationLifetime for RequestLease {
fn retain(&self) -> Box<dyn Send> {
// This is continuation of an already admitted request, not fresh admission.
// Retain its drain responsibility before transferring work to the DB owner;
// dropping/aborting the HTTP waiter must not allow deletion past live SQL.
let mut state = self
.gate
.state
.lock()
.expect("Workspace admission poisoned");
if self.callback {
state.callbacks += 1;
} else {
state.requests += 1;
}
Box::new(RequestLease {
gate: self.gate.clone(),
callback: self.callback,
})
}
}
impl Drop for RequestLease {
fn drop(&mut self) {
let mut state = self
@@ -0,0 +1,606 @@
//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs).
use super::*;
use crate::SqliteWorkspaceStore;
use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime};
use rusqlite::{Connection, ffi};
use std::{
ffi::c_void,
future::Future,
num::NonZeroUsize,
sync::{atomic::AtomicBool, mpsc},
thread::JoinHandle,
time::Duration as StdDuration,
};
use tokio::sync::oneshot;
const WORKSPACE: &str = "db-execution";
const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects";
const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2);
const WATCHDOG: StdDuration = StdDuration::from_secs(5);
async fn observed<T>(future: impl Future<Output = T>) -> T {
tokio::time::timeout(OBSERVE_LIMIT, future)
.await
.expect("request did not reach the expected DB/lifecycle state")
}
struct Fixture {
// Keep temporary authority files alive until all other fixture owners drop.
server: WorkspaceServerApi,
api: WorkspaceApi,
executor: DbExecutor,
_dir: tempfile::TempDir,
}
impl Fixture {
async fn new() -> Self {
let dir = tempfile::tempdir().unwrap();
let executor = DbExecutor::new(DbExecutorLimits {
workers: NonZeroUsize::new(1).unwrap(),
queued: NonZeroUsize::new(1).unwrap(),
})
.unwrap();
// Explicit startup config: never resolve the developer's live data dir.
let config = ServerConfig {
workspace_id: WORKSPACE.into(),
workspace_display_name: "DB execution fixture".into(),
workspace_created_at: "2026-01-01T00:00:00Z".into(),
workspace_execution_root: dir.path().into(),
database_path: dir.path().join("server.db"),
data_root: dir.path().join("data"),
embedded_runtime_store_root: dir.path().join("runtime"),
static_assets_dir: None,
auth: AuthConfig::Passkey {
rp_id: "localhost".into(),
origin: "http://localhost:5173".into(),
public_base_url: "http://localhost:5173".into(),
cookie_name: "db_execution_session".into(),
},
max_records: 100,
repositories: Vec::new(),
runtime_event_sources: Vec::new(),
remote_runtime_sources: Vec::new(),
backend_base_url: Some("http://localhost:8787".into()),
};
let store = Arc::new(
SqliteWorkspaceStore::open(&config.database_path)
.unwrap()
.with_db_executor(executor.clone()),
);
seed_test_registered_workspace(store.as_ref(), &config)
.await
.unwrap();
store
.upsert_user(&UserRecord {
user_id: "fixture-user".into(),
account_id: format!("account-{WORKSPACE}"),
handle: "fixture-user".into(),
display_name: "Fixture user".into(),
created_at: config.workspace_created_at.clone(),
updated_at: config.workspace_created_at.clone(),
})
.unwrap();
store
.create_browser_session(&crate::store::BrowserSessionRecord {
session_id: "fixture-session".into(),
token_hash: crate::auth::token_hash("fixture-browser-token"),
user_id: "fixture-user".into(),
created_at: config.workspace_created_at.clone(),
expires_at: "2999-01-01T00:00:00Z".into(),
revoked_at: None,
})
.unwrap();
let api = WorkspaceApi::new(config.clone(), store.clone())
.await
.unwrap();
let server = WorkspaceServerApi::new(config, store);
// Cache only the real inner router; avoid starting the unrelated hook.
server
.apis
.lock()
.await
.insert(WORKSPACE.into(), api.clone());
server
.routers
.lock()
.await
.insert(WORKSPACE.into(), build_inner_router(api.clone()));
executor.drain().await.unwrap();
Self {
server,
api,
executor,
_dir: dir,
}
}
fn request(method: &str, path: &str, body: serde_json::Value) -> Request {
Request::builder()
.method(method)
.uri(path)
.header(CONTENT_TYPE, "application/json")
.header("cookie", "db_execution_session=fixture-browser-token")
.header(ORIGIN, "http://localhost:5173")
.body(axum::body::Body::from(body.to_string()))
.unwrap()
}
fn subject_request(role: &str) -> Request {
Self::request(
"POST",
SUBJECTS,
serde_json::json!({ "role": role, "behavior_md": "" }),
)
}
fn dispatch(&self, request: Request) -> tokio::task::JoinHandle<Response> {
let server = self.server.clone();
tokio::spawn(async move { dispatch_workspace_request(State(server), request).await })
}
// A pre-authenticated generated router isolates handler admission from the
// earlier browser-session DB read. Full authentication/dispatch is exercised
// separately by the cancellation and warm-Feature tests below.
fn inner_router(&self) -> Router {
build_inner_router(self.api.clone()).layer(Extension(RequestActor {
user_id: "fixture-user".into(),
account_id: format!("account-{WORKSPACE}"),
handle: "fixture-user".into(),
display_name: "Fixture user".into(),
auth_method: ActorAuthMethod::BrowserSession,
}))
}
fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) {
let database = self
.api
.feature_storage
.open(&self.api.subjektiv_registration)
.unwrap();
// Initialize the trusted scope, then obtain SQLite's actual file path
// from the managed handle rather than guessing the Feature directory.
crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap();
let path = database
.with_connection(|conn| {
Ok(PathBuf::from(conn.query_row(
"SELECT file FROM pragma_database_list WHERE name = 'main'",
[],
|row| row.get::<_, String>(0),
)?))
})
.unwrap();
(database, path)
}
}
/// SQLite write ownership lives on an OS thread, including a bounded watchdog.
/// Drop releases and joins it even when an assertion panics on the async side.
struct WriteLock {
release: Option<mpsc::Sender<()>>,
thread: Option<JoinHandle<()>>,
released: Arc<AtomicBool>,
}
impl WriteLock {
fn acquire(path: PathBuf) -> Self {
let (ready, held) = mpsc::sync_channel(1);
let (release, permit) = mpsc::channel();
let released = Arc::new(AtomicBool::new(false));
let flag = released.clone();
let thread = std::thread::spawn(move || {
let conn = Connection::open(path).unwrap();
conn.execute_batch("BEGIN IMMEDIATE").unwrap();
ready.send(()).unwrap();
let _ = permit.recv_timeout(WATCHDOG);
conn.execute_batch("ROLLBACK").unwrap();
flag.store(true, Ordering::SeqCst);
});
held.recv_timeout(WATCHDOG)
.expect("SQLite lock holder did not acquire its transaction");
Self {
release: Some(release),
thread: Some(thread),
released,
}
}
fn release(&mut self) {
if let Some(release) = self.release.take() {
let _ = release.send(());
}
if let Some(thread) = self.thread.take() {
thread.join().unwrap();
}
}
}
impl Drop for WriteLock {
fn drop(&mut self) {
self.release();
}
}
struct BusyGate {
entered: Option<oneshot::Sender<()>>,
resume: mpsc::Receiver<()>,
}
/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's
/// userdata API can, without process globals or changing production hooks.
struct BusyObservation {
database: crate::FeatureDatabase,
gate: Box<BusyGate>,
resume: Option<mpsc::Sender<()>>,
}
impl BusyObservation {
fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) {
unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 {
// SAFETY: owned Box remains registered until Drop unregisters it,
// serialized by the managed connection mutex.
let gate = unsafe { &mut *context.cast::<BusyGate>() };
if let Some(entered) = gate.entered.take() {
let _ = entered.send(());
return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok());
}
0
}
let (entered, waiting) = oneshot::channel();
let (resume, permit) = mpsc::channel();
let mut gate = Box::new(BusyGate {
entered: Some(entered),
resume: permit,
});
database
.with_connection(|conn| {
// SAFETY: handle belongs to this connection; the Box is stable.
let code = unsafe {
ffi::sqlite3_busy_handler(
conn.handle(),
Some(busy),
(&mut *gate as *mut BusyGate).cast(),
)
};
assert_eq!(code, ffi::SQLITE_OK);
Ok(())
})
.unwrap();
(
Self {
database,
gate,
resume: Some(resume),
},
waiting,
)
}
fn resume(&mut self) {
if let Some(resume) = self.resume.take() {
let _ = resume.send(());
}
}
}
impl Drop for BusyObservation {
fn drop(&mut self) {
self.resume();
self.database
.with_connection(|conn| {
// SAFETY: unregister before the boxed userdata is freed.
unsafe {
ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut());
}
Ok(())
})
.unwrap();
let _ = &self.gate;
}
}
// A handler with one DB stage can expose its admission without adding a test
// hook to production. Polling its future to Pending proves run accepted it.
struct AdmissionObservation(Mutex<Option<oneshot::Sender<()>>>);
impl DbOperationLifetime for AdmissionObservation {
fn retain(&self) -> Box<dyn Send> {
if let Some(accepted) = self.0.lock().unwrap().take() {
let _ = accepted.send(());
}
Box::new(())
}
}
fn admission_observation() -> (Arc<dyn DbOperationLifetime>, oneshot::Receiver<()>) {
let (accepted, receiver) = oneshot::channel();
(
Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))),
receiver,
)
}
struct StateCheckSaturation {
executor: DbExecutor,
stages: std::sync::atomic::AtomicUsize,
releases: Arc<Mutex<Vec<mpsc::Sender<()>>>>,
}
impl DbOperationLifetime for StateCheckSaturation {
fn retain(&self) -> Box<dyn Send> {
if self.stages.fetch_add(1, Ordering::SeqCst) == 1 {
// Authentication has completed; saturate just before the lifecycle
// lookup is admitted. The callback's own dispatch admissions are
// distinct stages and do not recurse into this branch.
let (started, running) = mpsc::channel();
let (release, permit) = mpsc::channel();
self.executor
.dispatch(move || {
let _ = started.send(());
let _ = permit.recv_timeout(WATCHDOG);
})
.unwrap();
running
.recv_timeout(WATCHDOG)
.expect("state-check blocker did not start");
self.releases.lock().unwrap().push(release);
let (release, permit) = mpsc::channel();
self.executor
.dispatch(move || {
let _ = permit.recv_timeout(WATCHDOG);
})
.unwrap();
self.releases.lock().unwrap().push(release);
}
Box::new(())
}
}
struct ReleaseStateCheckBlockers(Arc<Mutex<Vec<mpsc::Sender<()>>>>);
impl Drop for ReleaseStateCheckBlockers {
fn drop(&mut self) {
self.0.lock().unwrap().clear();
}
}
#[tokio::test(flavor = "current_thread")]
async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() {
let fixture = Fixture::new().await;
let releases = Arc::new(Mutex::new(Vec::new()));
let release = ReleaseStateCheckBlockers(releases.clone());
let source = Arc::new(StateCheckSaturation {
executor: fixture.executor.clone(),
stages: std::sync::atomic::AtomicUsize::new(0),
releases,
});
let router = Router::new()
.route(
SUBJECTS,
axum::routing::post(|| async { StatusCode::NO_CONTENT }),
)
.layer(axum::middleware::from_fn_with_state(
fixture.api.clone(),
authorize_workspace_api_request,
));
let response = observed(crate::db_executor::with_operation_lifetime(
source,
router.oneshot(Fixture::subject_request("not-executed")),
))
.await
.unwrap();
drop(release);
observed(fixture.executor.drain()).await.unwrap();
let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await;
assert!(
!body.to_string().contains("Workspace is deleting"),
"{body}"
);
}
async fn runtime_progress_while_locked(lock: &WriteLock) {
let control = tokio::spawn(async { "control progressed" });
assert_eq!(observed(control).await.unwrap(), "control progressed");
// The timer is the contract, not a guessed wait for request completion.
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
assert!(
!lock.released.load(Ordering::SeqCst),
"SQLite watchdog released the lock before runtime progress"
);
}
async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value {
let status = response.status();
let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
.await
.unwrap();
assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes));
serde_json::from_slice(&bytes).expect("typed JSON response")
}
fn subject_count(path: &Path, role: &str) -> i64 {
Connection::open(path)
.unwrap()
.query_row(
"SELECT count(*) FROM subjects WHERE role = ?1",
[role],
|row| row.get(0),
)
.unwrap()
}
#[tokio::test(flavor = "current_thread")]
async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() {
let fixture = Fixture::new().await;
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
let (lifetime, accepted) = admission_observation();
let request = Fixture::request(
"POST",
"/api/w/db-execution/tickets",
serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(),
);
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
lifetime,
fixture.inner_router().oneshot(request),
));
assert!(futures::poll!(&mut response).is_pending());
observed(accepted).await.unwrap();
runtime_progress_while_locked(&lock).await;
assert!(
futures::poll!(&mut response).is_pending(),
"mutation responded before the SQLite lock was released"
);
let reader = Connection::open(&fixture.api.config.database_path).unwrap();
let count: i64 = reader
.query_row(
"SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1",
[WORKSPACE],
|row| row.get(0),
)
.unwrap();
assert_eq!(count, 0, "Ticket must not be visible before commit");
lock.release();
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
assert!(body["id"].is_string(), "{body}");
// Ticket dispatch can call Runtime callbacks, so its operation owner is
// the separate bounded host executor, not the store executor.
observed(host_io_executor().unwrap().drain()).await.unwrap();
fixture.executor.drain().await.unwrap();
let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap();
assert_eq!(count, 1, "one accepted HTTP mutation must commit once");
}
#[tokio::test(flavor = "current_thread")]
async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
drop(database); // FeatureStorage retains Weak handles: the route must cold-open.
let mut lock = WriteLock::acquire(path);
let (lifetime, accepted) = admission_observation();
let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null);
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
lifetime,
fixture.inner_router().oneshot(request),
));
assert!(futures::poll!(&mut response).is_pending());
observed(accepted).await.unwrap();
runtime_progress_while_locked(&lock).await;
assert!(
futures::poll!(&mut response).is_pending(),
"cold open must wait for its SQLite scope transaction"
);
lock.release();
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
assert_eq!(body["items"], serde_json::json!([]), "{body}");
fixture.executor.drain().await.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
let mut lock = WriteLock::acquire(path.clone());
let (mut busy, waiting) = BusyObservation::install(database);
let response = fixture.dispatch(Fixture::subject_request("locked-subject"));
observed(waiting).await.unwrap();
runtime_progress_while_locked(&lock).await;
assert!(
!response.is_finished(),
"HTTP response must not publish an uncommitted subject"
);
assert_eq!(subject_count(&path, "locked-subject"), 0);
lock.release();
busy.resume();
let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await;
assert_eq!(body["role"], "locked-subject", "{body}");
assert!(
body["memory_fingerprint"].is_string(),
"response assembly must finish in the DB operation: {body}"
);
fixture.executor.drain().await.unwrap();
assert_eq!(subject_count(&path, "locked-subject"), 1);
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
let mut lock = WriteLock::acquire(path.clone());
let (mut busy, waiting) = BusyObservation::install(database);
let response = fixture.dispatch(Fixture::subject_request("cancelled-subject"));
observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling.
response.abort();
assert!(observed(response).await.unwrap_err().is_cancelled());
let admission = fixture.server.admission_for_workspace(WORKSPACE).await;
let mut deletion = Box::pin(admission.lock_deletion());
assert!(
futures::poll!(&mut deletion).is_pending(),
"aborting HTTP must not drop the accepted SQL RequestLease"
);
assert!(
admission.admit(false).is_none(),
"deletion must fence fresh requests"
);
assert_eq!(subject_count(&path, "cancelled-subject"), 0);
lock.release();
busy.resume();
fixture.executor.drain().await.unwrap();
let deletion = observed(deletion).await;
assert_eq!(
subject_count(&path, "cancelled-subject"),
1,
"cancelled waiter must neither cancel nor retry accepted SQL"
);
drop(deletion);
assert!(
admission.admit(false).is_some(),
"deletion lease cleanup must reopen admission"
);
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
let mut lock = WriteLock::acquire(path.clone());
let (mut busy, waiting) = BusyObservation::install(database);
let running = fixture.dispatch(Fixture::subject_request("running-subject"));
observed(waiting).await.unwrap();
let (lifetime, accepted) = admission_observation();
let mut queued = Box::pin(crate::db_executor::with_operation_lifetime(
lifetime,
fixture
.inner_router()
.oneshot(Fixture::subject_request("queued-subject")),
));
assert!(futures::poll!(&mut queued).is_pending());
observed(accepted).await.unwrap();
drop(queued); // Accepted job still owns the only waiting slot.
let rejected = observed(
fixture
.inner_router()
.oneshot(Fixture::subject_request("rejected-subject")),
)
.await
.unwrap();
let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await;
let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap();
assert!(
error.message.contains("capacity"),
"typed overload error: {body}"
);
assert_eq!(subject_count(&path, "queued-subject"), 0);
assert_eq!(subject_count(&path, "rejected-subject"), 0);
let mut drained = Box::pin(fixture.executor.drain());
assert!(futures::poll!(&mut drained).is_pending());
lock.release();
busy.resume();
response_json(observed(running).await.unwrap(), StatusCode::CREATED).await;
observed(drained).await.unwrap();
assert_eq!(
subject_count(&path, "queued-subject"),
1,
"cancelled accepted mutation persists exactly once"
);
assert_eq!(
subject_count(&path, "rejected-subject"),
0,
"overload must not execute rejected SQL"
);
let retry = fixture.dispatch(Fixture::subject_request("after-drain"));
response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await;
fixture.executor.drain().await.unwrap();
assert_eq!(
subject_count(&path, "after-drain"),
1,
"drain must release actual-job capacity"
);
}
+6 -7
View File
@@ -51,14 +51,13 @@ async fn blocking<T: Send + 'static>(
mutation: bool,
action: impl FnOnce() -> workspace_drive::Result<T> + Send + 'static,
) -> DriveResult<T> {
tokio::task::spawn_blocking(action)
crate::db_executor::DbExecutor::shared()
.run(move || Ok::<_, Error>(action()))
.await
.map_err(|_| {
if mutation {
DriveApiError::new(DriveApiErrorCode::OutcomeUnknown)
} else {
failure(503)
}
.map_err(|error| match error {
Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503),
_ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown),
_ => failure(503),
})?
.map_err(|error| domain_error(error, mutation))
}
@@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> DriveApiError {
})
}
fn mutation_error(error: Error) -> DriveApiError {
if matches!(error, Error::Sqlite(_) | Error::Store(_)) {
if matches!(
error,
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
) {
fail(DriveApiErrorCode::OutcomeUnknown)
} else {
pre_error(error)
@@ -97,23 +100,27 @@ pub(super) async fn authorized_drive(
};
// Reject missing/read-only grants before upload body reception. This check
// is admission only: every later operation rechecks under its own lock.
tokio::task::spawn_blocking(move || {
drive.check_authorized(write)?;
Ok::<_, workspace_drive::Error>(drive)
})
.await
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
.map_err(|error| {
fail(match error {
workspace_drive::Error::Denied => DriveApiErrorCode::Denied,
workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => {
DriveApiErrorCode::Conflict
}
workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound,
workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid,
_ => DriveApiErrorCode::StorageUnavailable,
api.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
drive.check_authorized(write)?;
Ok::<_, workspace_drive::Error>(drive)
})())
})
.await
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
.map_err(|error| {
fail(match error {
workspace_drive::Error::Denied => DriveApiErrorCode::Denied,
workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => {
DriveApiErrorCode::Conflict
}
workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound,
workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid,
_ => DriveApiErrorCode::StorageUnavailable,
})
})
})
}
pub(super) async fn create_grant(
@@ -152,9 +159,10 @@ pub(super) async fn create_grant(
revoked_at: None,
};
let store = api.store.clone();
tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant))
api.store
.db_executor()
.run(move || store.create_workspace_drive_grant(&grant))
.await
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
.map_err(mutation_error)
}
@@ -175,13 +183,12 @@ pub(super) async fn revoke_grant(
actor.account_id.clone(),
);
let store = api.store.clone();
tokio::task::spawn_blocking(move || {
store.revoke_workspace_drive_grant(&workspace, &grant, &actor)
})
.await
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
.map_err(mutation_error)?
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
api.store
.db_executor()
.run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor))
.await
.map_err(mutation_error)?
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
}
fn validate_grant_id(id: &str) -> DriveResult<()> {
@@ -212,12 +219,14 @@ pub(super) async fn list_grants(
}
let workspace = workspace_id.to_owned();
let store = api.store.clone();
let mut grants = tokio::task::spawn_blocking(move || {
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
})
.await
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
.map_err(pre_error)?;
let mut grants = api
.store
.db_executor()
.run(move || {
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
})
.await
.map_err(pre_error)?;
let more = grants.len() > limit;
grants.truncate(limit);
let next_after = if more {
@@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> Failure {
fn mutation_error(error: Error) -> Failure {
// Storage/service failures at a mutation boundary may include a commit
// acknowledgement failure. Never describe those as a guaranteed rollback.
if matches!(&error, Error::Sqlite(_) | Error::Store(_)) {
if matches!(
&error,
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
) {
unknown()
} else {
pre_error(error)
@@ -186,9 +189,23 @@ pub(super) async fn get(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
) -> ConfigResult<Option<WorkspaceConfigAttachment>> {
let _guard = lock(api, worker).await?;
let grant = grant_for_worker(api, worker)?;
Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true)))
let guard = lock(api, worker).await?;
let executor = api.store.db_executor();
let api = api.clone();
let worker = worker.clone();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
let grant = grant_for_worker(&api, &worker)?;
Ok::<_, Failure>(
active_link(&api, &worker, &grant, None)?
.map(|link| attachment(&grant, &link, true)),
)
})())
})
.await
.map_err(pre_error)?
}
pub(super) async fn attach(
api: &WorkspaceApi,
@@ -198,46 +215,59 @@ pub(super) async fn attach(
if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) {
return Err(fail(400, "invalid_alias"));
}
let _guard = lock(api, worker).await?;
let grant = grant_for_worker(api, worker)?;
let access = request.access.unwrap_or(grant.access);
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
return Err(fail(403, "access_denied"));
}
if let Some(mut link) = active_link(api, worker, &grant, None)? {
// Same active alias/identity is idempotent; explicit write never silently downgrades.
if request.access == Some(Access::ReadWrite)
&& effective_access(&grant, &link) != Access::ReadWrite
{
return Err(fail(403, "access_denied"));
}
if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly {
link.capabilities = access_capabilities(access);
link = api
.store
.attach_worker_workdir(&link)
.map_err(mutation_error)?;
}
return Ok(attachment(&grant, &link, true));
}
let link = api
.store
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
connection_id: String::new(),
workspace_id: grant.workspace_id.clone(),
worker: worker.clone(),
workdir_id: grant.working_directory_id.clone(),
alias: ALIAS.into(),
capabilities: access_capabilities(access),
linked_at: now_registry_timestamp(),
unlinked_at: None,
let guard = lock(api, worker).await?;
let executor = api.store.db_executor();
let api = api.clone();
let worker = worker.clone();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
let grant = grant_for_worker(&api, &worker)?;
let access = request.access.unwrap_or(grant.access);
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
return Err(fail(403, "access_denied"));
}
if let Some(mut link) = active_link(&api, &worker, &grant, None)? {
// Same active alias/identity is idempotent; explicit write never silently downgrades.
if request.access == Some(Access::ReadWrite)
&& effective_access(&grant, &link) != Access::ReadWrite
{
return Err(fail(403, "access_denied"));
}
if access == Access::ReadOnly
&& effective_access(&grant, &link) != Access::ReadOnly
{
link.capabilities = access_capabilities(access);
link = api
.store
.attach_worker_workdir(&link)
.map_err(mutation_error)?;
}
return Ok(attachment(&grant, &link, true));
}
let link = api
.store
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
connection_id: String::new(),
workspace_id: grant.workspace_id.clone(),
worker: worker.clone(),
workdir_id: grant.working_directory_id.clone(),
alias: ALIAS.into(),
capabilities: access_capabilities(access),
linked_at: now_registry_timestamp(),
unlinked_at: None,
})
.map_err(mutation_error)?;
// No Runtime filesystem/session synchronization for a logical config target.
api.worker_projection
.refresh(&worker)
.map_err(|_| unknown())?;
Ok(attachment(&grant, &link, false))
})())
})
.map_err(mutation_error)?;
// No Runtime filesystem/session synchronization for a logical config target.
api.worker_projection
.refresh(worker)
.map_err(|_| unknown())?;
Ok(attachment(&grant, &link, false))
.await
.map_err(mutation_error)?
}
fn resolve(
api: &WorkspaceApi,
@@ -313,9 +343,22 @@ pub(super) async fn observe(
for path in &request.paths {
path_valid(path, true)?;
}
let _guard = lock(api, worker).await?;
let (grant, link) = resolve(api, worker, &request.connection_id)?;
let state = tree(api)?;
let guard = lock(api, worker).await?;
let resolution_api = api.clone();
let resolution_worker = worker.clone();
let connection_id = request.connection_id.clone();
let (grant, link, state, _guard) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
let state = tree(&resolution_api)?;
Ok::<_, Failure>((grant, link, state, guard))
})())
})
.await
.map_err(pre_error)??;
let schema_fingerprint = api
.config_schema_registry
.compose()
@@ -427,9 +470,22 @@ pub(super) async fn read(
request: WorkspaceConfigReadRequest,
) -> ConfigResult<WorkspaceConfigReadResponse> {
path_valid(&request.path, false)?;
let _guard = lock(api, worker).await?;
let (grant, link) = resolve(api, worker, &request.connection_id)?;
let state = tree(api)?;
let guard = lock(api, worker).await?;
let resolution_api = api.clone();
let resolution_worker = worker.clone();
let connection_id = request.connection_id.clone();
let (grant, link, state, _guard) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
let state = tree(&resolution_api)?;
Ok::<_, Failure>((grant, link, state, guard))
})())
})
.await
.map_err(pre_error)??;
let schema_fingerprint = api
.config_schema_registry
.compose()
@@ -482,12 +538,25 @@ pub(super) async fn commit(
server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?,
}
}
let _guard = lock(api, worker).await?;
let (grant, link) = resolve(api, worker, &request.connection_id)?;
if effective_access(&grant, &link) != Access::ReadWrite {
return Err(fail(403, "access_denied"));
}
let state = tree(api)?;
let guard = lock(api, worker).await?;
let resolution_api = api.clone();
let resolution_worker = worker.clone();
let connection_id = request.connection_id.clone();
let (grant, link, state, guard) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
if effective_access(&grant, &link) != Access::ReadWrite {
return Err(fail(403, "access_denied"));
}
let state = tree(&resolution_api)?;
Ok::<_, Failure>((grant, link, state, guard))
})())
})
.await
.map_err(pre_error)??;
let schema_fingerprint = api
.config_schema_registry
.compose()
@@ -501,17 +570,19 @@ pub(super) async fn commit(
// Evaluation/semantic validation are pure preparation. Bound this stage
// without ever dispatching persistence in a timed-out background task.
let preparation_api = api.clone();
let preparation = tokio::task::spawn_blocking(move || {
prepare_workspace_config_tree(
let executor = api.store.db_executor();
let preparation = executor.run(move || {
// Timeout/cancellation releases exclusion only after actual preparation ends.
let candidate = prepare_workspace_config_tree(
&preparation_api,
&preparation_api.config.workspace_id,
&canonical,
)
)?;
Ok::<_, Error>((candidate, guard))
});
let candidate = tokio::time::timeout(DEADLINE, preparation)
let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation)
.await
.map_err(|_| fail(503, "evaluation_timeout"))?
.map_err(|_| fail(500, "evaluation_unavailable"))?
.map_err(pre_error)?;
if candidate.base_digest != state.snapshot.digest
|| candidate.base_toolchain_fingerprint != state.contract.fingerprint
@@ -520,15 +591,78 @@ pub(super) async fn commit(
{
return Err(fail(409, "stale_validator"));
}
// Recheck durable authority before the first auxiliary/canonical effects.
let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?;
if current_grant != grant || current_link.capabilities != link.capabilities {
return Err(fail(409, "authority_changed"));
let persistence_api = api.clone();
let worker = worker.clone();
let connection_id = request.connection_id.clone();
let expected_grant = grant.clone();
let expected_link = link.clone();
let (state, publication) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (current_grant, current_link) =
resolve(&persistence_api, &worker, &connection_id)?;
if current_grant != expected_grant
|| current_link.capabilities != expected_link.capabilities
{
return Err(fail(409, "authority_changed"));
}
// Preserve the shared helper's ordering and atomic CAS, but never
// run its remote Runtime publication on the sole DB worker.
project_repository_access_candidate(
&*persistence_api.store,
&persistence_api.repository_secrets,
&persistence_api.config.workspace_id,
&candidate,
)
.map_err(|_| unknown())?;
let state = persistence_api
.config_store
.commit_evaluated_workspace_config(
&persistence_api.config.workspace_id,
&candidate,
)
.map_err(|_| unknown())?;
let publication = persistence_api
.prompt_projection_cache
.resolve(&persistence_api.config.workspace_id, &state)
.ok()
.and_then(|projection| {
// Dispatch from the actual persistence owner even if its
// waiter was cancelled. Runtime I/O owns exclusion, not
// a DB worker or an open SQL transaction.
let (complete, publication) = tokio::sync::oneshot::channel();
let runtime = persistence_api.runtime.clone();
let admission = host_io_executor().and_then(|executor| {
executor.dispatch(move || {
let _guard = guard;
let _diagnostics = runtime
.observe_workspace_prompt_projection((*projection).clone());
let _ = complete.send(());
})
});
match admission {
Ok(()) => Some(publication),
Err(error) => {
tracing::warn!(
workspace_id = %persistence_api.config.workspace_id,
%error,
"Workspace config committed but Runtime publication was not admitted"
);
None
}
}
});
Ok((state, publication))
})())
})
.await
.map_err(mutation_error)??;
if let Some(publication) = publication {
// Publication is best effort and cannot turn an acknowledged CAS into an error.
let _ = publication.await;
}
// From this boundary repository credential projection may have side effects.
// Never falsely claim rollback or automatically replay a failed/uncertain save.
let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate)
.map_err(|_| unknown())?;
Ok(WorkspaceConfigCommitResponse {
validator: validator(&grant, &link, &state, &schema_fingerprint, ""),
digest: state.snapshot.digest,
@@ -544,37 +678,59 @@ pub(super) async fn create_grant(
.await
.map_err(pre_error)?;
let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id);
let _guard = lock(api, &worker).await?;
let summary = api
.runtime
.worker(&worker)
.map_err(|_| fail(404, "worker_not_found"))?;
let guard = lock(api, &worker).await?;
let runtime = api.runtime.clone();
let lookup_worker = worker.clone();
let (summary, guard) = host_io_executor()
.map_err(pre_error)?
.run(move || {
Ok::<_, Error>((|| {
let summary = runtime
.worker(&lookup_worker)
.map_err(|_| fail(404, "worker_not_found"))?;
Ok::<_, Failure>((summary, guard))
})())
})
.await
.map_err(pre_error)??;
if summary.workspace.workspace_id.as_deref() != Some(workspace_id) {
return Err(fail(403, "access_denied"));
}
if let Some(grant) = api
.store
.current_workspace_config_grant(workspace_id, &worker)
.map_err(pre_error)?
{
if grant.access != request.access {
return Err(fail(409, "grant_exists_revoke_first"));
}
return Ok(grant);
}
let grant = WorkspaceConfigGrantResponse {
grant_id: Uuid::now_v7().to_string(),
workspace_id: workspace_id.into(),
runtime_id: worker.runtime_id,
worker_id: worker.worker_id,
working_directory_id: Uuid::now_v7().to_string(),
access: request.access,
revoked: false,
};
api.store
.create_workspace_config_grant(&grant, &actor.account_id)
.map_err(mutation_error)?;
Ok(grant)
let executor = api.store.db_executor();
let api = api.clone();
let workspace_id = workspace_id.to_owned();
let account_id = actor.account_id.clone();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
if let Some(grant) = api
.store
.current_workspace_config_grant(&workspace_id, &worker)
.map_err(pre_error)?
{
if grant.access != request.access {
return Err(fail(409, "grant_exists_revoke_first"));
}
return Ok(grant);
}
let grant = WorkspaceConfigGrantResponse {
grant_id: Uuid::now_v7().to_string(),
workspace_id,
runtime_id: worker.runtime_id,
worker_id: worker.worker_id,
working_directory_id: Uuid::now_v7().to_string(),
access: request.access,
revoked: false,
};
api.store
.create_workspace_config_grant(&grant, &account_id)
.map_err(mutation_error)?;
Ok(grant)
})())
})
.await
.map_err(mutation_error)?
}
pub(super) async fn revoke_grant(
api: &WorkspaceApi,
@@ -585,39 +741,60 @@ pub(super) async fn revoke_grant(
require_workspace_owner(api, workspace_id, actor, "Workspace config grants")
.await
.map_err(pre_error)?;
let lookup_api = api.clone();
let lookup_workspace_id = workspace_id.to_owned();
let lookup_grant_id = grant_id.to_owned();
let mut grant = api
.store
.get_workspace_config_grant(workspace_id, grant_id)
.db_executor()
.run(move || {
lookup_api
.store
.get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id)
})
.await
.map_err(pre_error)?
.ok_or_else(|| fail(404, "not_found"))?;
let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id);
let _guard = lock(api, &worker).await?;
api.store
.revoke_workspace_config_grant(workspace_id, grant_id)
.map_err(mutation_error)?;
grant.revoked = true;
// Revocation is durable before unlink; a failed cleanup cannot restore access.
if let Some(link) = api
.store
.list_workdir_worker_links(workspace_id, &grant.working_directory_id)
.map_err(|_| unknown())?
.into_iter()
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
{
api.store
.detach_worker_workdir_connection(
workspace_id,
&worker,
&link.alias,
&link.connection_id,
&now_registry_timestamp(),
)
.map_err(|_| unknown())?;
}
api.worker_projection
.refresh(&worker)
.map_err(|_| unknown())?;
Ok(grant)
let guard = lock(api, &worker).await?;
let executor = api.store.db_executor();
let api = api.clone();
let workspace_id = workspace_id.to_owned();
let grant_id = grant_id.to_owned();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
api.store
.revoke_workspace_config_grant(&workspace_id, &grant_id)
.map_err(mutation_error)?;
grant.revoked = true;
// Revocation is durable before unlink; a failed cleanup cannot restore access.
if let Some(link) = api
.store
.list_workdir_worker_links(&workspace_id, &grant.working_directory_id)
.map_err(|_| unknown())?
.into_iter()
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
{
api.store
.detach_worker_workdir_connection(
&workspace_id,
&worker,
&link.alias,
&link.connection_id,
&now_registry_timestamp(),
)
.map_err(|_| unknown())?;
}
api.worker_projection
.refresh(&worker)
.map_err(|_| unknown())?;
Ok(grant)
})())
})
.await
.map_err(mutation_error)?
}
pub(super) fn summary(
@@ -655,7 +832,11 @@ pub(super) fn summary(
mod tests {
use super::*;
#[test]
fn workspace_config_mutation_storage_failures_are_conservatively_unknown() {
fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() {
assert_eq!(
mutation_error(Error::DbOperationOutcomeUnknown).classification,
Classification::Unknown
);
assert_eq!(
mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification,
Classification::Unknown
@@ -803,9 +803,9 @@ mod workspace_config_integration {
.supports(workdir::WorkdirSessionCapability::Command)
);
let lock = current_worker_session_lock(api, &fixture.worker);
let _guard = lock.lock().await;
let _guard = Arc::new(lock.lock_owned().await);
assert!(
open_current_worker_workdir_session_locked(api, &fixture.worker, &link)
open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard)
.await
.is_err()
);
+461 -218
View File
@@ -1123,6 +1123,10 @@ impl WorkspaceResourceKind {
#[async_trait]
pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
fn db_executor(&self) -> crate::db_executor::DbExecutor {
crate::db_executor::DbExecutor::shared()
}
async fn schema_version(&self) -> Result<i64>;
fn resource_key(
&self,
@@ -2179,6 +2183,7 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
#[derive(Clone)]
pub struct SqliteWorkspaceStore {
conn: Arc<Mutex<Connection>>,
db_executor: crate::db_executor::DbExecutor,
}
impl SqliteWorkspaceStore {
@@ -2222,9 +2227,20 @@ impl SqliteWorkspaceStore {
prepare_connection(&conn)?;
Ok(Self {
conn: Arc::new(Mutex::new(conn)),
db_executor: crate::db_executor::DbExecutor::shared(),
})
}
/// Override execution limits for a hermetic fixture; subsequent clones share it.
pub fn with_db_executor(mut self, executor: crate::db_executor::DbExecutor) -> Self {
self.db_executor = executor;
self
}
pub fn db_executor(&self) -> crate::db_executor::DbExecutor {
self.db_executor.clone()
}
pub(crate) fn with_conn<T>(&self, f: impl FnOnce(&Connection) -> Result<T>) -> Result<T> {
let conn = self
.conn
@@ -3794,8 +3810,16 @@ fn insert_workspace_runtime_binding_audit(
#[async_trait]
impl ControlPlaneStore for SqliteWorkspaceStore {
fn db_executor(&self) -> crate::db_executor::DbExecutor {
self.db_executor.clone()
}
async fn schema_version(&self) -> Result<i64> {
self.with_conn(current_schema_version)
let store = self.clone();
self.db_executor
.run(move || store.with_conn(current_schema_version))
.await
}
fn resource_key(
@@ -3927,100 +3951,114 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
}
async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
#[cfg(test)]
if record.owner_account_id == "owner-account" {
// Unrelated store tests use this explicit fixture identity. Keep the
// production owner contract strict while giving those fixtures a real
// User Account row instead of reviving ownerless Workspace setup.
tx.execute(
"INSERT OR IGNORE INTO accounts (
account_id, kind, handle, display_name, created_at, updated_at
) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)",
params![record.owner_account_id.as_str(), record.created_at.as_str()],
)?;
}
let owner_kind = tx
.query_row(
"SELECT kind FROM accounts WHERE account_id = ?1",
params![record.owner_account_id.as_str()],
|row| row.get::<_, String>(0),
)
.optional()?;
if owner_kind.as_deref() != Some("user") {
return Err(Error::Store(format!(
"Workspace owner `{}` must reference an existing User Account",
record.owner_account_id
)));
}
let current_owner = tx
.query_row(
"SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1",
params![record.workspace_id.as_str()],
|row| row.get::<_, String>(0),
)
.optional()?;
if current_owner
.as_deref()
.is_some_and(|owner| owner != record.owner_account_id)
{
return Err(Error::Store(format!(
"Workspace `{}` owner is immutable through upsert",
record.workspace_id
)));
}
tx.execute(
r#"INSERT INTO workspaces (
workspace_id, owner_account_id, display_name, state, created_at, updated_at
) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(workspace_id) DO UPDATE SET
display_name = excluded.display_name,
state = excluded.state,
updated_at = excluded.updated_at"#,
params![
record.workspace_id,
record.owner_account_id,
record.display_name,
record.state,
record.created_at,
record.updated_at,
],
)?;
tx.execute(
r#"INSERT OR IGNORE INTO workspace_memory_settings (
workspace_id, language, created_at, updated_at
) VALUES (?1, 'English', ?2, ?3)"#,
params![record.workspace_id, record.created_at, record.updated_at],
)?;
tx.execute(
r#"INSERT OR IGNORE INTO workspace_signing_identities (
workspace_id, key_id, algorithm, public_key, public_key_fingerprint,
private_material_ref, state, created_at, provisioned_at, updated_at
) VALUES (
?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL,
'workspace-signing/' || ?1 || '/ed25519-v1',
'pending_provisioning', ?2, NULL, ?3
)"#,
params![record.workspace_id, record.created_at, record.updated_at],
)?;
tx.commit()?;
Ok(())
})?;
self.materialize_workspace_config(&record.workspace_id, &record.created_at)
let store = self.clone();
let record = record.clone();
self.db_executor
.run(move || {
let store = &store;
let record = &record;
store.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
#[cfg(test)]
if record.owner_account_id == "owner-account" {
// Unrelated store tests use this explicit fixture identity. Keep the
// production owner contract strict while giving those fixtures a real
// User Account row instead of reviving ownerless Workspace setup.
tx.execute(
"INSERT OR IGNORE INTO accounts (
account_id, kind, handle, display_name, created_at, updated_at
) VALUES (?1, 'user', 'owner-account', 'Owner Account', ?2, ?2)",
params![record.owner_account_id.as_str(), record.created_at.as_str()],
)?;
}
let owner_kind = tx
.query_row(
"SELECT kind FROM accounts WHERE account_id = ?1",
params![record.owner_account_id.as_str()],
|row| row.get::<_, String>(0),
)
.optional()?;
if owner_kind.as_deref() != Some("user") {
return Err(Error::Store(format!(
"Workspace owner `{}` must reference an existing User Account",
record.owner_account_id
)));
}
let current_owner = tx
.query_row(
"SELECT owner_account_id FROM workspaces WHERE workspace_id = ?1",
params![record.workspace_id.as_str()],
|row| row.get::<_, String>(0),
)
.optional()?;
if current_owner
.as_deref()
.is_some_and(|owner| owner != record.owner_account_id)
{
return Err(Error::Store(format!(
"Workspace `{}` owner is immutable through upsert",
record.workspace_id
)));
}
tx.execute(
r#"INSERT INTO workspaces (
workspace_id, owner_account_id, display_name, state, created_at, updated_at
) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(workspace_id) DO UPDATE SET
display_name = excluded.display_name,
state = excluded.state,
updated_at = excluded.updated_at"#,
params![
record.workspace_id,
record.owner_account_id,
record.display_name,
record.state,
record.created_at,
record.updated_at,
],
)?;
tx.execute(
r#"INSERT OR IGNORE INTO workspace_memory_settings (
workspace_id, language, created_at, updated_at
) VALUES (?1, 'English', ?2, ?3)"#,
params![record.workspace_id, record.created_at, record.updated_at],
)?;
tx.execute(
r#"INSERT OR IGNORE INTO workspace_signing_identities (
workspace_id, key_id, algorithm, public_key, public_key_fingerprint,
private_material_ref, state, created_at, provisioned_at, updated_at
) VALUES (
?1, 'WK-' || lower(hex(randomblob(16))), 'ed25519', NULL, NULL,
'workspace-signing/' || ?1 || '/ed25519-v1',
'pending_provisioning', ?2, NULL, ?3
)"#,
params![record.workspace_id, record.created_at, record.updated_at],
)?;
tx.commit()?;
Ok(())
})?;
store.materialize_workspace_config(&record.workspace_id, &record.created_at)
})
.await
}
async fn get_workspace(&self, workspace_id: &str) -> Result<Option<WorkspaceRecord>> {
self.with_conn(|conn| {
conn.query_row(
r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at
FROM workspaces WHERE workspace_id = ?1"#,
params![workspace_id],
read_workspace_record,
)
.optional()
.map_err(Error::from)
})
let store = self.clone();
let workspace_id = workspace_id.to_owned();
self.db_executor.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
store.with_conn(|conn| {
conn.query_row(
r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at
FROM workspaces WHERE workspace_id = ?1"#,
params![workspace_id],
read_workspace_record,
)
.optional()
.map_err(Error::from)
})
}).await
}
async fn update_workspace_display_name(
@@ -4029,58 +4067,68 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
expected_updated_at: &str,
display_name: &str,
) -> Result<Option<WorkspaceRecord>> {
validate_identifier("workspace_id", workspace_id)?;
validate_non_empty("expected_updated_at", expected_updated_at)?;
validate_non_empty("display_name", display_name)?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let current = tx
.query_row(
r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at
FROM workspaces WHERE workspace_id = ?1"#,
params![workspace_id],
read_workspace_record,
)
.optional()?;
let Some(current) = current else {
tx.commit()?;
return Ok(None);
};
if current.updated_at != expected_updated_at {
tx.commit()?;
return Ok(None);
}
if current.display_name == display_name {
tx.commit()?;
return Ok(Some(current));
}
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let expected_updated_at = expected_updated_at.to_owned();
let display_name = display_name.to_owned();
self.db_executor.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let expected_updated_at = expected_updated_at.as_str();
let display_name = display_name.as_str();
validate_identifier("workspace_id", workspace_id)?;
validate_non_empty("expected_updated_at", expected_updated_at)?;
validate_non_empty("display_name", display_name)?;
store.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let current = tx
.query_row(
r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at
FROM workspaces WHERE workspace_id = ?1"#,
params![workspace_id],
read_workspace_record,
)
.optional()?;
let Some(current) = current else {
tx.commit()?;
return Ok(None);
};
if current.updated_at != expected_updated_at {
tx.commit()?;
return Ok(None);
}
if current.display_name == display_name {
tx.commit()?;
return Ok(Some(current));
}
let now = chrono::Utc::now();
let updated_at = chrono::DateTime::parse_from_rfc3339(&current.updated_at)
.ok()
.map(|previous| previous.with_timezone(&chrono::Utc))
.filter(|previous| *previous >= now)
.map(|previous| previous + chrono::Duration::nanoseconds(1))
.unwrap_or(now)
.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true);
let changed = tx.execute(
r#"UPDATE workspaces
SET display_name = ?3, updated_at = ?4
WHERE workspace_id = ?1 AND updated_at = ?2"#,
params![workspace_id, expected_updated_at, display_name, updated_at],
)?;
if changed != 1 {
let now = chrono::Utc::now();
let updated_at = chrono::DateTime::parse_from_rfc3339(&current.updated_at)
.ok()
.map(|previous| previous.with_timezone(&chrono::Utc))
.filter(|previous| *previous >= now)
.map(|previous| previous + chrono::Duration::nanoseconds(1))
.unwrap_or(now)
.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true);
let changed = tx.execute(
r#"UPDATE workspaces
SET display_name = ?3, updated_at = ?4
WHERE workspace_id = ?1 AND updated_at = ?2"#,
params![workspace_id, expected_updated_at, display_name, updated_at],
)?;
if changed != 1 {
tx.commit()?;
return Ok(None);
}
let updated = WorkspaceRecord {
display_name: display_name.to_string(),
updated_at,
..current
};
tx.commit()?;
return Ok(None);
}
let updated = WorkspaceRecord {
display_name: display_name.to_string(),
updated_at,
..current
};
tx.commit()?;
Ok(Some(updated))
})
Ok(Some(updated))
})
}).await
}
fn create_workspace_bootstrap(
@@ -4600,7 +4648,17 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
workspace_id: &str,
runtime_id: &str,
) -> Result<Option<WorkspaceRuntimeBinding>> {
SqliteWorkspaceStore::get_workspace_runtime_binding(self, workspace_id, runtime_id)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
SqliteWorkspaceStore::get_workspace_runtime_binding(store, workspace_id, runtime_id)
})
.await
}
async fn list_workspace_runtime_bindings(
@@ -4608,7 +4666,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
workspace_id: &str,
include_revoked: bool,
) -> Result<Vec<WorkspaceRuntimeBinding>> {
SqliteWorkspaceStore::list_workspace_runtime_bindings(self, workspace_id, include_revoked)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
SqliteWorkspaceStore::list_workspace_runtime_bindings(
store,
workspace_id,
include_revoked,
)
})
.await
}
async fn has_other_active_workspace_runtime_binding(
@@ -4616,11 +4686,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
workspace_id: &str,
runtime_id: &str,
) -> Result<bool> {
SqliteWorkspaceStore::has_other_active_workspace_runtime_binding(
self,
workspace_id,
runtime_id,
)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
SqliteWorkspaceStore::has_other_active_workspace_runtime_binding(
store,
workspace_id,
runtime_id,
)
})
.await
}
async fn delete_workspace_runtime_binding(
@@ -4628,7 +4708,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
workspace_id: &str,
runtime_id: &str,
) -> Result<bool> {
SqliteWorkspaceStore::delete_workspace_runtime_binding(self, workspace_id, runtime_id)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
SqliteWorkspaceStore::delete_workspace_runtime_binding(
store,
workspace_id,
runtime_id,
)
})
.await
}
async fn reserve_runtime_removal(
@@ -4640,22 +4734,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
expected_binding_id: &str,
config_digest: &str,
) -> Result<RuntimeRemovalReservation> {
SqliteWorkspaceStore::reserve_runtime_removal(
self,
workspace_id,
runtime_id,
operation_id,
request_fingerprint,
expected_binding_id,
config_digest,
)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
let operation_id = operation_id.to_owned();
let request_fingerprint = request_fingerprint.to_owned();
let expected_binding_id = expected_binding_id.to_owned();
let config_digest = config_digest.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
let operation_id = operation_id.as_str();
let request_fingerprint = request_fingerprint.as_str();
let expected_binding_id = expected_binding_id.as_str();
let config_digest = config_digest.as_str();
SqliteWorkspaceStore::reserve_runtime_removal(
store,
workspace_id,
runtime_id,
operation_id,
request_fingerprint,
expected_binding_id,
config_digest,
)
})
.await
}
async fn get_runtime_removal(
&self,
operation_id: &str,
) -> Result<Option<RuntimeRemovalOperation>> {
SqliteWorkspaceStore::get_runtime_removal(self, operation_id)
let store = self.clone();
let operation_id = operation_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let operation_id = operation_id.as_str();
SqliteWorkspaceStore::get_runtime_removal(store, operation_id)
})
.await
}
async fn mark_runtime_removal_failed(
@@ -4663,14 +4783,36 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
operation_id: &str,
failure_category: &str,
) -> Result<RuntimeRemovalOperation> {
SqliteWorkspaceStore::mark_runtime_removal_failed(self, operation_id, failure_category)
let store = self.clone();
let operation_id = operation_id.to_owned();
let failure_category = failure_category.to_owned();
self.db_executor
.run(move || {
let store = &store;
let operation_id = operation_id.as_str();
let failure_category = failure_category.as_str();
SqliteWorkspaceStore::mark_runtime_removal_failed(
store,
operation_id,
failure_category,
)
})
.await
}
async fn commit_runtime_binding_removal(
&self,
operation_id: &str,
) -> Result<RuntimeRemovalOperation> {
SqliteWorkspaceStore::commit_runtime_binding_removal(self, operation_id)
let store = self.clone();
let operation_id = operation_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let operation_id = operation_id.as_str();
SqliteWorkspaceStore::commit_runtime_binding_removal(store, operation_id)
})
.await
}
async fn complete_runtime_removal(
@@ -4678,18 +4820,34 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
operation_id: &str,
runtime_registration_removed: bool,
) -> Result<RuntimeRemovalOperation> {
SqliteWorkspaceStore::complete_runtime_removal(
self,
operation_id,
runtime_registration_removed,
)
let store = self.clone();
let operation_id = operation_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let operation_id = operation_id.as_str();
SqliteWorkspaceStore::complete_runtime_removal(
store,
operation_id,
runtime_registration_removed,
)
})
.await
}
async fn list_resumable_runtime_removals(
&self,
workspace_id: &str,
) -> Result<Vec<RuntimeRemovalOperation>> {
SqliteWorkspaceStore::list_resumable_runtime_removals(self, workspace_id)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
SqliteWorkspaceStore::list_resumable_runtime_removals(store, workspace_id)
})
.await
}
async fn upsert_workspace_runtime_binding_record(
@@ -4697,7 +4855,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
record: WorkspaceRuntimeBinding,
replace: bool,
) -> Result<WorkspaceRuntimeBindingUpsert> {
SqliteWorkspaceStore::upsert_workspace_runtime_binding(self, record, replace)
let store = self.clone();
self.db_executor
.run(move || {
let store = &store;
SqliteWorkspaceStore::upsert_workspace_runtime_binding(store, record, replace)
})
.await
}
async fn revoke_workspace_runtime_binding_record(
@@ -4706,12 +4871,24 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
runtime_id: &str,
revoked_at: &str,
) -> Result<bool> {
SqliteWorkspaceStore::revoke_workspace_runtime_binding(
self,
workspace_id,
runtime_id,
revoked_at,
)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
let revoked_at = revoked_at.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
let revoked_at = revoked_at.as_str();
SqliteWorkspaceStore::revoke_workspace_runtime_binding(
store,
workspace_id,
runtime_id,
revoked_at,
)
})
.await
}
async fn put_workspace_runtime_binding_key(
@@ -4720,12 +4897,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
expected_binding_id: Option<&str>,
actor_account_id: &str,
) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> {
SqliteWorkspaceStore::put_workspace_runtime_binding_key(
self,
record,
expected_binding_id,
actor_account_id,
)
let store = self.clone();
let expected_binding_id = expected_binding_id.map(str::to_owned);
let actor_account_id = actor_account_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let expected_binding_id = expected_binding_id.as_deref();
let actor_account_id = actor_account_id.as_str();
SqliteWorkspaceStore::put_workspace_runtime_binding_key(
store,
record,
expected_binding_id,
actor_account_id,
)
})
.await
}
async fn update_workspace_runtime_binding_metadata(
@@ -4736,14 +4923,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
base_url: &str,
updated_at: &str,
) -> Result<WorkspaceRuntimeBinding> {
SqliteWorkspaceStore::update_workspace_runtime_binding_metadata(
self,
workspace_id,
runtime_id,
display_name,
base_url,
updated_at,
)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
let display_name = display_name.to_owned();
let base_url = base_url.to_owned();
let updated_at = updated_at.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
let display_name = display_name.as_str();
let base_url = base_url.as_str();
let updated_at = updated_at.as_str();
SqliteWorkspaceStore::update_workspace_runtime_binding_metadata(
store,
workspace_id,
runtime_id,
display_name,
base_url,
updated_at,
)
})
.await
}
async fn revoke_workspace_runtime_binding_key(
@@ -4754,14 +4957,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
actor_account_id: &str,
revoked_at: &str,
) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> {
SqliteWorkspaceStore::revoke_workspace_runtime_binding_key(
self,
workspace_id,
runtime_id,
expected_binding_id,
actor_account_id,
revoked_at,
)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
let expected_binding_id = expected_binding_id.to_owned();
let actor_account_id = actor_account_id.to_owned();
let revoked_at = revoked_at.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
let expected_binding_id = expected_binding_id.as_str();
let actor_account_id = actor_account_id.as_str();
let revoked_at = revoked_at.as_str();
SqliteWorkspaceStore::revoke_workspace_runtime_binding_key(
store,
workspace_id,
runtime_id,
expected_binding_id,
actor_account_id,
revoked_at,
)
})
.await
}
async fn list_workspace_runtime_binding_audit(
@@ -4770,12 +4989,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
runtime_id: &str,
limit: usize,
) -> Result<Vec<WorkspaceRuntimeBindingAuditRecord>> {
SqliteWorkspaceStore::list_workspace_runtime_binding_audit(
self,
workspace_id,
runtime_id,
limit,
)
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
SqliteWorkspaceStore::list_workspace_runtime_binding_audit(
store,
workspace_id,
runtime_id,
limit,
)
})
.await
}
async fn consume_worker_mutation_source_jti(
@@ -4787,21 +5016,35 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
now_seconds: u64,
consumed_at: &str,
) -> Result<bool> {
self.with_conn(|conn| {
let transaction = conn.unchecked_transaction()?;
transaction.execute(
"DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1",
params![now_seconds],
)?;
let inserted = transaction.execute(
r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis (
workspace_id, runtime_id, jti, expires_at, consumed_at
) VALUES (?1, ?2, ?3, ?4, ?5)"#,
params![workspace_id, runtime_id, jti, expires_at, consumed_at],
)?;
transaction.commit()?;
Ok(inserted == 1)
})
let store = self.clone();
let workspace_id = workspace_id.to_owned();
let runtime_id = runtime_id.to_owned();
let jti = jti.to_owned();
let consumed_at = consumed_at.to_owned();
self.db_executor
.run(move || {
let store = &store;
let workspace_id = workspace_id.as_str();
let runtime_id = runtime_id.as_str();
let jti = jti.as_str();
let consumed_at = consumed_at.as_str();
store.with_conn(|conn| {
let transaction = conn.unchecked_transaction()?;
transaction.execute(
"DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1",
params![now_seconds],
)?;
let inserted = transaction.execute(
r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis (
workspace_id, runtime_id, jti, expires_at, consumed_at
) VALUES (?1, ?2, ?3, ?4, ?5)"#,
params![workspace_id, runtime_id, jti, expires_at, consumed_at],
)?;
transaction.commit()?;
Ok(inserted == 1)
})
})
.await
}
fn plan_worker_removal(
+434 -21
View File
@@ -5,7 +5,7 @@ use chrono::Utc;
use protocol::subscription::{
EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot,
};
use tokio::sync::broadcast;
use tokio::sync::{Semaphore, broadcast};
use tokio::task::JoinHandle;
use crate::runtime_subscription::{
@@ -59,7 +59,14 @@ pub struct WorkerProjectionService {
events: broadcast::Sender<WorkerProjectionEvent>,
ordering: Mutex<()>,
attach_gate: tokio::sync::Mutex<()>,
tasks: Mutex<HashMap<String, JoinHandle<()>>>,
tasks: Mutex<HashMap<String, RuntimeProjectionLifetime>>,
}
struct RuntimeProjectionLifetime {
consumer: Option<JoinHandle<()>>,
// The permit belongs to the actual DB closure, not its cancellable async waiter.
// Keep this owner in the map even if attachment initialization is cancelled.
operations: Arc<Semaphore>,
}
impl WorkerProjectionService {
@@ -75,17 +82,89 @@ impl WorkerProjectionService {
})
}
/// Stops consumers but does not claim that accepted database work has finished.
/// Async shutdown callers must use `shutdown_async` to drain accepted operations.
pub fn shutdown(&self) {
for (_, task) in self
for lifetime in self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.drain()
.values()
{
task.abort();
if let Some(task) = &lifetime.consumer {
task.abort();
}
}
}
pub async fn shutdown_async(&self) {
let _attach = self.attach_gate.lock().await;
self.shutdown();
let runtime_ids = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.keys()
.cloned()
.collect::<Vec<_>>();
for runtime_id in runtime_ids {
self.drain_runtime(&runtime_id).await;
}
self.tasks
.lock()
.expect("Worker projection task lock poisoned")
.clear();
}
async fn drain_runtime(&self, runtime_id: &str) {
let previous = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.get_mut(runtime_id)
.map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone()));
if let Some((consumer, operations)) = previous {
if let Some(consumer) = consumer {
consumer.abort();
let _ = consumer.await;
}
// Aborting the waiter cannot abort a queued/running synchronous job.
// Cancellation here leaves the gate in the map for the next drain.
let _drained = operations
.acquire_owned()
.await
.expect("operation gate closed");
}
}
// Caller holds attach_gate. Install ownership before any initialization await.
async fn replace_runtime(&self, runtime_id: &str) -> Arc<Semaphore> {
self.drain_runtime(runtime_id).await;
let operations = Arc::new(Semaphore::new(1));
self.tasks
.lock()
.expect("Worker projection task lock poisoned")
.insert(
runtime_id.to_string(),
RuntimeProjectionLifetime {
consumer: None,
operations: operations.clone(),
},
);
operations
}
pub async fn subscribe_ordered_async(
self: &Arc<Self>,
limit: usize,
) -> crate::Result<WorkerProjectionSubscription> {
let service = self.clone();
self.store
.db_executor()
.run(move || service.subscribe_ordered(limit))
.await
}
pub fn subscribe_ordered(&self, limit: usize) -> crate::Result<WorkerProjectionSubscription> {
let _ordering = self
.ordering
@@ -125,19 +204,10 @@ impl WorkerProjectionService {
broker: &RuntimeSubscriptionBroker,
runtime_id: &str,
) -> Result<(), RuntimeSubscriptionBrokerError> {
// Serialize replacement and wait for the old consumer to stop before the new
// lifetime can apply its snapshot. Late notifications can therefore never
// cross a Runtime ownership boundary.
// Serialize replacement, stop admission by aborting the consumer, and then
// drain ownership held by the actual SQL jobs before the new snapshot.
let _attach = self.attach_gate.lock().await;
let previous = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.remove(runtime_id);
if let Some(previous) = previous {
previous.abort();
let _ = previous.await;
}
let operations = self.replace_runtime(runtime_id).await;
let mut subscription =
broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?;
@@ -148,12 +218,25 @@ impl WorkerProjectionService {
break;
};
let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. });
if let Err(error) = self.apply_broker_event(runtime_id, event) {
if let Err(error) = self
.apply_broker_event_owned_async(runtime_id, event, operations.clone())
.await
{
tracing::warn!(
runtime_id,
error = %error,
"failed to initialize durable Worker projection; restarting from a fresh snapshot"
);
if matches!(
error,
crate::Error::DbOperationOutcomeUnknown
| crate::Error::DbExecutorUnavailable
| crate::Error::DbCapacityExceeded
) {
// Do not automatically replay a possibly committed mutation or
// spin on executor overload; attachment must be retried explicitly.
return Err(RuntimeSubscriptionBrokerError::Closed);
}
subscription.restart()?;
continue;
}
@@ -169,12 +252,27 @@ impl WorkerProjectionService {
let Some(service) = service.upgrade() else {
return;
};
if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) {
if let Err(error) = service
.apply_broker_event_owned_async(
task_runtime_id.as_str(),
event,
operations.clone(),
)
.await
{
tracing::warn!(
runtime_id = %task_runtime_id,
error = %error,
"failed to update durable Worker projection; restarting from a fresh snapshot"
);
if matches!(
error,
crate::Error::DbOperationOutcomeUnknown
| crate::Error::DbExecutorUnavailable
| crate::Error::DbCapacityExceeded
) {
return;
}
if subscription.restart().is_err() {
return;
}
@@ -184,10 +282,61 @@ impl WorkerProjectionService {
self.tasks
.lock()
.expect("Worker projection task lock poisoned")
.insert(runtime_id, task);
.get_mut(&runtime_id)
.expect("Runtime lifetime installed before initialization")
.consumer = Some(task);
Ok(())
}
#[cfg(test)]
async fn seed_observation_async(
self: &Arc<Self>,
runtime_id: &str,
worker: &protocol::subscription::SubscriptionWorker,
observed_at: &str,
) -> crate::Result<()> {
let service = self.clone();
let runtime_id = runtime_id.to_string();
let worker = worker.clone();
let observed_at = observed_at.to_string();
self.store
.db_executor()
.run(move || service.seed_observation(&runtime_id, &worker, &observed_at))
.await
}
pub async fn refresh_async(self: &Arc<Self>, worker: &RuntimeWorkerRef) -> crate::Result<()> {
let service = self.clone();
let worker = worker.clone();
self.store
.db_executor()
.run(move || service.refresh(&worker))
.await
}
async fn apply_broker_event_owned_async(
self: &Arc<Self>,
runtime_id: &str,
event: BrokerSubscriptionEvent,
operations: Arc<Semaphore>,
) -> crate::Result<()> {
let operation = operations
.acquire_owned()
.await
.expect("operation gate closed");
let service = self.clone();
let runtime_id = runtime_id.to_string();
self.store
.db_executor()
.run(move || {
// The permit must outlive SQL commit AND ordered publication, even
// when the executor result receiver has been dropped by cancellation.
let _operation = operation;
service.apply_broker_event(&runtime_id, event)
})
.await
}
pub fn seed_observation(
&self,
runtime_id: &str,
@@ -313,7 +462,11 @@ mod tests {
tempfile::TempDir,
) {
let temp = tempfile::tempdir().unwrap();
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap());
let store = Arc::new(
SqliteWorkspaceStore::open(temp.path().join("workspace.db"))
.unwrap()
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
);
store
.upsert_account(&AccountRecord {
account_id: "owner".to_string(),
@@ -355,6 +508,266 @@ mod tests {
}
}
fn observation(
state: protocol::subscription::SubscriptionWorkerState,
) -> protocol::subscription::SubscriptionWorker {
protocol::subscription::SubscriptionWorker {
restore_observation_token: None,
worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(),
runtime_id: Some("runtime-a".to_string()),
resource_key: None,
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
worker_state: None,
state,
has_running_internal_workers: false,
workspace_id: Some(WORKSPACE_ID.to_string()),
display_name: None,
profile: None,
job: None,
workdir_attachments: Vec::new(),
}
}
// Real second-connection contention; the controller releases automatically on
// failure, so a regression that blocks the Tokio thread cannot hang the suite.
struct WriteLock {
release: Option<std::sync::mpsc::Sender<()>>,
controller: Option<std::thread::JoinHandle<()>>,
}
impl WriteLock {
fn acquire(path: std::path::PathBuf) -> Self {
let (release, released) = std::sync::mpsc::channel();
let (locked, ready) = std::sync::mpsc::channel();
let controller = std::thread::spawn(move || {
let connection = rusqlite::Connection::open(path).unwrap();
connection.execute_batch("BEGIN IMMEDIATE").unwrap();
locked.send(()).unwrap();
let _ = released.recv_timeout(std::time::Duration::from_secs(3));
connection.execute_batch("ROLLBACK").unwrap();
});
ready.recv().unwrap();
Self {
release: Some(release),
controller: Some(controller),
}
}
fn release(&mut self) {
if let Some(release) = self.release.take() {
let _ = release.send(());
}
}
}
impl Drop for WriteLock {
fn drop(&mut self) {
self.release();
if let Some(controller) = self.controller.take() {
controller.join().unwrap();
}
}
}
async fn wait_for_sql(service: &WorkerProjectionService) {
tokio::time::timeout(std::time::Duration::from_secs(1), async {
// With the second connection holding the write lock, a held ordering
// gate identifies the accepted synchronous mutation, not a fake future.
while service.ordering.try_lock().is_ok() {
tokio::task::yield_now().await;
}
})
.await
.expect("projection job did not enter synchronous SQL");
}
#[tokio::test(flavor = "current_thread")]
async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload()
{
use protocol::subscription::SubscriptionWorkerState;
let (store, service, temp) = fixture().await;
store
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
.unwrap();
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
assert_eq!(subscription.take_snapshot().len(), 1);
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
let publishing_service = service.clone();
let publication = tokio::spawn(async move {
publishing_service
.seed_observation_async(
"runtime-a",
&observation(SubscriptionWorkerState::Idle),
"2",
)
.await
});
wait_for_sql(&service).await;
let started = std::time::Instant::now();
let (control, controlled) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
control.send("control progressed").unwrap();
});
let (_, control_result) = tokio::join!(
tokio::time::sleep(std::time::Duration::from_millis(20)),
controlled,
);
assert_eq!(control_result.unwrap(), "control progressed");
assert!(
started.elapsed() < std::time::Duration::from_secs(1),
"SQLite blocked the current-thread executor"
);
assert!(matches!(
subscription.try_recv(),
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
));
lock.release();
publication.await.unwrap().unwrap();
let committed = subscription.recv().await.unwrap();
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
let snapshot = snapshot.take_snapshot();
assert_eq!(
committed.changes,
vec![WorkerCatalogChange::Upsert(snapshot[0].clone())]
);
assert_eq!(
snapshot[0].observation.as_ref().unwrap().worker.state,
SubscriptionWorkerState::Idle
);
service
.seed_observation_async(
"runtime-a",
&observation(SubscriptionWorkerState::Running),
"3",
)
.await
.unwrap();
// Delayed readers receive the first transaction's payload, not a re-query
// of mutable authority after a later commit.
let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else {
panic!("expected upsert");
};
assert_eq!(
first.observation.as_ref().unwrap().worker.state,
SubscriptionWorkerState::Idle
);
service
.refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a"))
.await
.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() {
use crate::runtime_subscription::BrokerSubscriptionEvent;
use protocol::subscription::SubscriptionWorkerState;
let (store, service, temp) = fixture().await;
store
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
.unwrap();
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
subscription.take_snapshot();
let operations = service.replace_runtime("runtime-a").await;
let old_operations = operations.clone();
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
let old_service = service.clone();
let consumer = tokio::spawn(async move {
old_service
.apply_broker_event_owned_async(
"runtime-a",
BrokerSubscriptionEvent::Event {
payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted {
worker: observation(SubscriptionWorkerState::Running),
},
},
operations,
)
.await
.unwrap();
});
service
.tasks
.lock()
.unwrap()
.get_mut("runtime-a")
.unwrap()
.consumer = Some(consumer);
wait_for_sql(&service).await;
let replacing_service = service.clone();
let mut replacement = tokio::spawn(async move {
let _attach = replacing_service.attach_gate.lock().await;
replacing_service.replace_runtime("runtime-a").await
});
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
.await
.is_err()
);
replacement.abort();
assert!(replacement.await.unwrap_err().is_cancelled());
assert_eq!(
old_operations.available_permits(),
0,
"cancelled drain lost accepted SQL ownership"
);
// A second replacement must still observe the old gate after cancellation.
let replacing_service = service.clone();
let mut replacement = tokio::spawn(async move {
let _attach = replacing_service.attach_gate.lock().await;
let operations = replacing_service.replace_runtime("runtime-a").await;
replacing_service
.apply_broker_event_owned_async(
"runtime-a",
BrokerSubscriptionEvent::Snapshot {
snapshot: protocol::subscription::SubscriptionSnapshot::Workers {
workers: vec![observation(SubscriptionWorkerState::Idle)],
},
},
operations,
)
.await
});
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
.await
.is_err()
);
assert!(matches!(
subscription.try_recv(),
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
));
lock.release();
replacement.await.unwrap().unwrap();
assert_eq!(old_operations.available_permits(), 1);
let old_commit = subscription.recv().await.unwrap();
let new_commit = subscription.recv().await.unwrap();
for (commit, expected) in [
(old_commit, SubscriptionWorkerState::Running),
(new_commit, SubscriptionWorkerState::Idle),
] {
let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else {
panic!("expected one committed upsert");
};
assert_eq!(record.observation.as_ref().unwrap().worker.state, expected);
}
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
assert_eq!(
snapshot.take_snapshot()[0]
.observation
.as_ref()
.unwrap()
.worker
.state,
SubscriptionWorkerState::Idle
);
assert!(matches!(
subscription.try_recv(),
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
));
service.shutdown_async().await;
assert!(service.tasks.lock().unwrap().is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn snapshot_registration_is_atomic_with_concurrent_publication() {
let (store, service, _temp) = fixture().await;
+32 -10
View File
@@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError {
Replay,
#[error("source Worker is not a current member of this Workspace Runtime catalog")]
WorkerCatalogMembership,
#[error("source proof authority is unavailable")]
Unavailable,
#[error("source proof authority failed: {0}")]
Authority(String),
}
impl From<crate::Error> for WorkerMutationSourceProofError {
fn from(error: crate::Error) -> Self {
match error {
crate::Error::DbCapacityExceeded
| crate::Error::DbExecutorUnavailable
| crate::Error::DbOperationOutcomeUnknown
| crate::Error::Sqlite(_) => Self::Unavailable,
other => Self::Authority(other.to_string()),
}
}
}
pub async fn verify_worker_remove_source(
api: &WorkspaceApi,
proof: PresentedWorkerMutationSourceProof<'_>,
@@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with(
let trusted = store
.get_workspace_runtime_binding(&config.workspace_id, &unverified.iss)
.await
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?
.map_err(WorkerMutationSourceProofError::from)?
.filter(|record| record.revoked_at.is_none())
.ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?;
let public_key = trusted.public_key.as_str();
@@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with(
runtime_id: claims.iss.clone(),
worker_id: claims.worker_id.clone(),
};
let member = store
.get_worker_registry(&config.workspace_id, &worker)
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
if member.is_none() {
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
}
let workspace_id = config.workspace_id.clone();
let catalog_store = Arc::clone(store);
store
.require_current_worker_singleton_owner(&config.workspace_id, &worker)
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
.db_executor()
.run(move || {
let member = catalog_store
.get_worker_registry(&workspace_id, &worker)
.map_err(WorkerMutationSourceProofError::from)?;
if member.is_none() {
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
}
catalog_store
.require_current_worker_singleton_owner(&workspace_id, &worker)
.map_err(WorkerMutationSourceProofError::from)?;
Ok(())
})
.await?;
let consumed_at = chrono::Utc::now().to_rfc3339();
let consumed = store
@@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with(
&consumed_at,
)
.await
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
.map_err(WorkerMutationSourceProofError::from)?;
if !consumed {
return Err(WorkerMutationSourceProofError::Replay);
}
@@ -1,3 +1,5 @@
use std::sync::Arc;
use worker_runtime::auth::{
RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims,
RuntimeRequestSourceExpectation, decode_runtime_request_source_claims,
@@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError {
pub(crate) exp: Option<i64>,
pub(crate) now: Option<i64>,
pub(crate) time_delta_seconds: Option<u64>,
/// Operational unavailability is not a proof or principal rejection.
pub(crate) availability: bool,
}
impl RuntimeRequestProofError {
pub fn is_availability_error(&self) -> bool {
self.availability
}
fn unavailable(mut self, reason: &'static str) -> Self {
self.reason = reason;
self.availability = true;
self
}
fn undecoded() -> Self {
Self {
reason: "invalid",
@@ -47,6 +61,7 @@ impl RuntimeRequestProofError {
exp: None,
now: None,
time_delta_seconds: None,
availability: false,
}
}
@@ -106,6 +121,18 @@ impl RuntimeRequestProofError {
}
}
impl From<crate::Error> for RuntimeRequestProofError {
fn from(error: crate::Error) -> Self {
let reason = match error {
crate::Error::DbCapacityExceeded => "db_capacity_exceeded",
crate::Error::DbExecutorUnavailable => "db_executor_unavailable",
crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown",
_ => "database_authority_unavailable",
};
Self::undecoded().unavailable(reason)
}
}
/// Domain-separated fixed-length fingerprints, not raw request-derived IDs.
/// Operators can correlate a known ID by applying this same function.
pub fn diagnostic_id_hash(id: &str) -> String {
@@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof(
body_digest: &str,
) -> Result<VerifiedRuntimeRequestSource, RuntimeRequestProofError> {
verify_runtime_request_source_proof_with_store(
api.store.as_ref(),
Arc::clone(&api.store),
&api.config,
proof,
workspace_id,
@@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof(
#[allow(clippy::too_many_arguments)]
pub async fn verify_runtime_request_source_proof_with_store(
store: &dyn ControlPlaneStore,
store: Arc<dyn ControlPlaneStore>,
config: &ServerConfig,
proof: &str,
workspace_id: &str,
@@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store(
#[allow(clippy::too_many_arguments)]
pub(crate) async fn verify_runtime_request_source_proof_with_clock(
store: &dyn ControlPlaneStore,
store: Arc<dyn ControlPlaneStore>,
config: &ServerConfig,
proof: &str,
workspace_id: &str,
@@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
let unverified = decode_runtime_request_source_claims(proof)
.map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?;
let diagnostic = RuntimeRequestProofError::decoded(&unverified);
let audience = remote_audience(config, workspace_id)
.map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?;
let audience = remote_audience(config, workspace_id).map_err(|_| {
diagnostic
.clone()
.unavailable("backend_audience_unavailable")
})?;
let trusted = store
.get_workspace_runtime_binding(workspace_id, &unverified.iss)
.await
.map_err(|_| {
diagnostic
.clone()
.reason("runtime_trust_authority_unavailable")
.unavailable("runtime_trust_authority_unavailable")
})?
.filter(|record| record.revoked_at.is_none())
.ok_or_else(|| {
@@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
&consumed_at,
)
.await
.map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))?
.map_err(|_| {
diagnostic
.clone()
.unavailable("replay_authority_unavailable")
})?
{
return Err(diagnostic.reason("replay"));
}
@@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
runtime_id: claims.iss.clone(),
worker_id: worker_id.to_owned(),
};
let member = store
.get_worker_registry(workspace_id, &worker)
.map_err(|_| {
diagnostic
.clone()
.reason("worker_catalog_authority_unavailable")
})?;
let reserved = store
.has_active_worker_create_reservation(workspace_id, &worker)
.map_err(|_| {
diagnostic
.clone()
.reason("worker_reservation_authority_unavailable")
})?;
if member.is_none() && !reserved {
return Err(diagnostic.reason("worker_catalog_membership"));
}
store
.require_current_worker_singleton_owner(workspace_id, &worker)
.map_err(|_| {
diagnostic
.clone()
.reason("worker_singleton_authority_denied")
let workspace_id = workspace_id.to_owned();
let executor = store.db_executor();
let stage_diagnostic = diagnostic.clone();
// Only synchronous catalog authority lives in this closure. The trust
// and JTI async wrappers above already offload through the executor.
executor
.run(move || {
let member = store
.get_worker_registry(&workspace_id, &worker)
.map_err(|_| {
stage_diagnostic
.clone()
.unavailable("worker_catalog_authority_unavailable")
})?;
let reserved = store
.has_active_worker_create_reservation(&workspace_id, &worker)
.map_err(|_| {
stage_diagnostic
.clone()
.unavailable("worker_reservation_authority_unavailable")
})?;
if member.is_none() && !reserved {
return Err(stage_diagnostic.reason("worker_catalog_membership"));
}
store
.require_current_worker_singleton_owner(&workspace_id, &worker)
.map_err(|error| {
if matches!(
error,
crate::Error::DbCapacityExceeded
| crate::Error::DbExecutorUnavailable
| crate::Error::DbOperationOutcomeUnknown
| crate::Error::Sqlite(_)
) {
stage_diagnostic
.clone()
.unavailable("worker_singleton_authority_unavailable")
} else {
stage_diagnostic
.clone()
.reason("worker_singleton_authority_denied")
}
})?;
Ok(())
})
.await
.map_err(|error: RuntimeRequestProofError| {
if error.verification == "unverified" && error.is_availability_error() {
diagnostic.clone().unavailable(error.reason)
} else {
error
}
})?;
}
Ok(VerifiedRuntimeRequestSource {
@@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
authentication_binding_id: trusted.binding_id,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::{
AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode,
WorkspaceRuntimeBinding,
};
use std::time::Duration;
use worker_runtime::auth::{
BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner,
request_body_digest,
};
const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa";
const NOW: i64 = 1_700_000_000;
const PATH: &str = "/api/runtime/v1/resources/fetch";
// rusqlite's busy handler accepts a function pointer, not a closure. Each
// fixture owns a dedicated single-worker executor, so this thread-local
// handshake is isolated from all parallel tests and product connections.
thread_local! {
static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell<Option<(
tokio::sync::oneshot::Sender<()>,
std::sync::mpsc::Receiver<()>,
)>> = const { std::cell::RefCell::new(None) };
}
fn acknowledge_sqlite_contention(_attempt: i32) -> bool {
SQLITE_BUSY_HANDSHAKE.with(|slot| {
let Some((blocked, retry)) = slot.borrow_mut().take() else {
return false;
};
let _ = blocked.send(());
// The test releases SQLite's writer lock before allowing retry.
// Dropping the sender on panic also releases this worker promptly.
retry.recv_timeout(Duration::from_secs(3)).is_ok()
})
}
async fn fixture(
root: &std::path::Path,
) -> (Arc<SqliteWorkspaceStore>, ServerConfig, String, String) {
let db_path = root.join("auth.db");
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
workers: std::num::NonZeroUsize::new(1).unwrap(),
queued: std::num::NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let store = Arc::new(
SqliteWorkspaceStore::open(&db_path)
.unwrap()
.with_db_executor(executor),
);
store
.upsert_account(&AccountRecord {
account_id: "account-auth".into(),
kind: "user".into(),
handle: "auth".into(),
display_name: "Auth".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
let workspace = WorkspaceRecord {
workspace_id: WORKSPACE.into(),
owner_account_id: "account-auth".into(),
display_name: "Auth".into(),
state: "active".into(),
created_at: "1".into(),
updated_at: "1".into(),
};
store.upsert_workspace(&workspace).await.unwrap();
let config = ServerConfig {
workspace_id: WORKSPACE.into(),
workspace_display_name: "Auth".into(),
workspace_created_at: "1".into(),
workspace_execution_root: root.to_owned(),
database_path: db_path,
data_root: root.to_owned(),
embedded_runtime_store_root: root.join("runtime"),
static_assets_dir: None,
auth: crate::server::AuthConfig::Passkey {
rp_id: "localhost".into(),
origin: "http://localhost".into(),
public_base_url: "http://localhost".into(),
cookie_name: "session".into(),
},
max_records: 10,
repositories: vec![],
runtime_event_sources: vec![],
remote_runtime_sources: vec![],
backend_base_url: Some("server-auth".into()),
};
let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap();
store
.upsert_workspace_runtime_binding_record(
WorkspaceRuntimeBinding {
workspace_id: WORKSPACE.into(),
runtime_id: "runtime-auth".into(),
display_name: "Auth".into(),
base_url: "http://127.0.0.1:1".into(),
public_key: identity.public_key.clone(),
public_key_fingerprint: String::new(),
binding_id: String::new(),
authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer,
created_at: "1".into(),
updated_at: "1".into(),
revoked_at: None,
},
false,
)
.await
.unwrap();
let binding = ControlPlaneStore::get_workspace_runtime_binding(
store.as_ref(),
WORKSPACE,
"runtime-auth",
)
.await
.unwrap()
.unwrap();
let proof = RuntimeRequestSourceSigner::from_identity(&identity)
.issue(
"server-auth",
WORKSPACE,
None,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
b"{}",
NOW,
30,
)
.unwrap();
(store, config, proof, binding.binding_id)
}
#[tokio::test(flavor = "current_thread")]
async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() {
let root = tempfile::tempdir().unwrap();
let (store, config, proof, binding_id) = fixture(root.path()).await;
// A real independent SQLite writer prevents JTI insertion while trust
// reads remain possible. Rollback on Drop also releases this on panic.
let mut writer = rusqlite::Connection::open(&config.database_path).unwrap();
let lock = writer
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.unwrap();
let (blocked, contention_reached) = tokio::sync::oneshot::channel();
let (retry, retry_rx) = std::sync::mpsc::channel();
let setup_store = store.clone();
store
.db_executor()
.run(move || -> crate::Result<()> {
SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx)));
setup_store.with_conn(|conn| {
conn.busy_handler(Some(acknowledge_sqlite_contention))?;
Ok(())
})
})
.await
.unwrap();
let request_store = store.clone();
let request_config = config.clone();
let request_proof = proof.clone();
let task = tokio::spawn(async move {
verify_runtime_request_source_proof_with_clock(
request_store,
&request_config,
&request_proof,
WORKSPACE,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
&request_body_digest(b"{}"),
&|| NOW,
)
.await
});
tokio::time::timeout(Duration::from_secs(1), contention_reached)
.await
.expect(
"SQLite JTI persistence must report real writer contention without blocking Tokio",
)
.unwrap();
// The busy callback acknowledges an actual SQLite write blocked by the
// independent writer, then holds retry until explicitly released.
assert!(
!task.is_finished(),
"JTI must remain pending while SQLite writer is locked"
);
let control = tokio::spawn(async { "control-responsive" });
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), control)
.await
.unwrap()
.unwrap(),
"control-responsive"
);
// This timer is the behavior being tested, not a guessed wait for SQL.
tokio::time::timeout(
Duration::from_secs(1),
tokio::time::sleep(Duration::from_millis(1)),
)
.await
.unwrap();
assert!(
!task.is_finished(),
"timer must advance while JTI remains blocked"
);
lock.rollback().unwrap();
retry.send(()).unwrap();
let source = tokio::time::timeout(Duration::from_secs(3), task)
.await
.unwrap()
.unwrap()
.unwrap();
assert_eq!(source.runtime_id, "runtime-auth");
assert_eq!(source.authentication_binding_id, binding_id);
let replay = verify_runtime_request_source_proof_with_clock(
store,
&config,
&proof,
WORKSPACE,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
&request_body_digest(b"{}"),
&|| NOW,
)
.await
.unwrap_err();
assert_eq!(replay.reason, "replay");
assert!(!replay.is_availability_error());
}
#[tokio::test]
async fn saturated_database_is_availability_failure_not_invalid_proof() {
let root = tempfile::tempdir().unwrap();
let (store, config, proof, _) = fixture(root.path()).await;
// Fill the isolated executor with a blocked job and queued jobs. No
// scheduling sleeps: the worker explicitly acknowledges its start.
let executor = store.db_executor();
let (started, started_rx) = tokio::sync::oneshot::channel();
let (release, release_rx) = std::sync::mpsc::channel();
let blocker_executor = executor.clone();
let blocker = tokio::spawn(async move {
blocker_executor
.run(move || -> crate::Result<()> {
let _ = started.send(());
release_rx.recv().unwrap();
Ok(())
})
.await
});
tokio::time::timeout(Duration::from_secs(1), started_rx)
.await
.unwrap()
.unwrap();
let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) }));
assert!(futures::poll!(queued.as_mut()).is_pending());
let outcome = tokio::time::timeout(
Duration::from_secs(1),
verify_runtime_request_source_proof_with_clock(
store,
&config,
&proof,
WORKSPACE,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
&request_body_digest(b"{}"),
&|| NOW,
),
)
.await;
// Release before assertions so a failed contract never strands a worker.
release.send(()).unwrap();
blocker.await.unwrap().unwrap();
queued.await.unwrap();
let error = outcome
.expect("capacity rejection must not wait for queued SQL")
.unwrap_err();
assert_eq!(error.reason, "runtime_trust_authority_unavailable");
assert!(error.is_availability_error());
}
}
@@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost {
impl WorkspaceDriveHost {
pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result<Self> {
let storage = storage.clone();
let registration = tokio::task::spawn_blocking(move || Drive::register(&storage))
.await
.map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))?
.map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?;
let registration = crate::db_executor::DbExecutor::shared()
.run(move || {
Drive::register(&storage).map_err(|error| {
Error::Store(format!("failed to register Drive storage: {error}"))
})
})
.await?;
Ok(Self {
registration,
purged: AtomicBool::new(false),
@@ -80,21 +83,21 @@ impl WorkspaceDriveHost {
}
let storage = storage.clone();
let registration = self.registration.clone();
let drive = tokio::task::spawn_blocking(move || {
reject_symlink_ancestors(&root)?;
// Domain IMMEDIATE transactions lock the attached server authority
// as well as Drive metadata. Deletion reservation and cached-handle
// operations therefore serialize with other server DB writes.
Drive::open_with_workspace_authority(
&storage,
&registration,
&root,
&authority_database,
)
.map_err(Error::from)
})
.await
.map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??;
let drive = crate::db_executor::DbExecutor::shared()
.run(move || {
reject_symlink_ancestors(&root)?;
// Domain IMMEDIATE transactions lock the attached server authority
// as well as Drive metadata. Deletion reservation and cached-handle
// operations therefore serialize with other server DB writes.
Drive::open_with_workspace_authority(
&storage,
&registration,
&root,
&authority_database,
)
.map_err(Error::from)
})
.await?;
*cached = Some(drive.clone());
Ok(drive)
}
@@ -109,9 +112,9 @@ impl WorkspaceDriveHost {
return Ok(());
}
let drive = self.open(storage, root, authority_database).await?;
tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from))
crate::db_executor::DbExecutor::shared()
.run(move || drive.fence().map_err(Error::from))
.await
.map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))?
}
pub(crate) async fn purge(
@@ -124,20 +127,20 @@ impl WorkspaceDriveHost {
return Ok(());
}
let drive = self.open(storage, root.clone(), authority_database).await?;
tokio::task::spawn_blocking(move || -> Result<()> {
reject_symlink_ancestors(&root)?;
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
drive.purge()?;
// Remove the empty blob root, never the data root or another Workspace.
// Missing roots are successful retries; other failures retain FeatureStorage.
match std::fs::remove_dir_all(&root) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
}
})
.await
.map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??;
crate::db_executor::DbExecutor::shared()
.run(move || -> Result<()> {
reject_symlink_ancestors(&root)?;
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
drive.purge()?;
// Remove the empty blob root, never the data root or another Workspace.
// Missing roots are successful retries; other failures retain FeatureStorage.
match std::fs::remove_dir_all(&root) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
}
})
.await?;
// A later signing-material/finalization failure can retry after the
// feature DB has been closed and removed by FeatureStorage.delete.
self.purged.store(true, Ordering::Release);
@@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription<S>(
worker_id,
runtime_id: Some(runtime_id),
} => {
let connection = match WorkspaceWorker::resolve(
let connection = match WorkspaceWorker::resolve_async(
&api,
&runtime_id,
worker_id.as_str(),
) {
)
.await
{
Ok(worker) => worker
.connect_protocol(&context)
.await
@@ -366,7 +368,8 @@ async fn run_workspace_workers(
) {
let Ok(mut projection) = api
.worker_projection
.subscribe_ordered(api.config.max_records.max(1))
.subscribe_ordered_async(api.config.max_records.max(1))
.await
else {
let _ = send_rejected(
&outbound,
@@ -377,11 +380,22 @@ async fn run_workspace_workers(
.await;
return;
};
let mut workers = projection
.take_snapshot()
.into_iter()
.filter_map(|record| project_registry_worker(&api, record))
.collect::<Vec<_>>();
let Ok(mut workers) = project_registry_workers_async(
api.store.clone(),
api.config.workspace_id.clone(),
projection.take_snapshot(),
)
.await
else {
let _ = send_rejected(
&outbound,
request_id,
SubscriptionRejectionCode::Internal,
"failed to enrich durable Worker projection".to_string(),
)
.await;
return;
};
sort_workers(&mut workers);
if send_frame(
&outbound,
@@ -406,7 +420,29 @@ async fn run_workspace_workers(
for change in event.changes {
let payload = match change {
crate::worker_projection::WorkerProjectionChange::Upsert(record) => {
let Some(worker) = project_registry_worker(&api, record) else {
let workers = project_registry_workers_async(
api.store.clone(),
api.config.workspace_id.clone(),
vec![record],
)
.await;
let Ok(mut workers) = workers else {
// Admission failure cannot silently lose a live
// update: terminate and require a fresh snapshot.
let _ = send_frame(
&outbound,
SubscriptionFrame::new(SubscriptionFramePayload::Event(
SubscriptionEvent::SubscriptionClosed {
subscription_id: subscription_id.clone(),
code: SubscriptionTerminationCode::Lagged,
message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(),
},
)),
)
.await;
return;
};
let Some(worker) = workers.pop() else {
continue;
};
SubscriptionEventPayload::WorkerUpserted { worker }
@@ -450,8 +486,28 @@ async fn run_workspace_workers(
}
}
async fn project_registry_workers_async(
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
workspace_id: String,
records: Vec<crate::store::WorkerRegistryProjectionRecord>,
) -> crate::Result<Vec<SubscriptionWorker>> {
store
.db_executor()
.run(move || {
// Keep the committed Worker payload; only the existing Workspace-scoped
// workdir/history/repository enrichment reads occur inside this job.
Ok(records
.into_iter()
.filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record))
.collect())
})
.await
}
// Synchronous helpers stay synchronous; they run only inside the job above.
fn project_registry_worker(
api: &WorkspaceApi,
store: &dyn crate::store::ControlPlaneStore,
workspace_id: &str,
record: crate::store::WorkerRegistryProjectionRecord,
) -> Option<SubscriptionWorker> {
let runtime_id = record.registry.worker.runtime_id.clone();
@@ -491,29 +547,25 @@ fn project_registry_worker(
attempt_id: job.attempt_id,
purpose: job.purpose,
});
if !project_working_directory(api, &record.registry.worker, &mut worker) {
if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) {
return None;
}
Some(worker)
}
fn project_working_directory(
api: &WorkspaceApi,
store: &dyn crate::store::ControlPlaneStore,
workspace_id: &str,
worker_ref: &RuntimeWorkerRef,
worker: &mut SubscriptionWorker,
) -> bool {
let Ok(links) = api
.store
.list_worker_workdir_links(&api.config.workspace_id, worker_ref)
else {
let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else {
return false;
};
if !links.is_empty() {
let mut attachments = Vec::with_capacity(links.len());
for link in links {
let Ok(Some(workdir)) = api
.store
.get_workdir_registry(&api.config.workspace_id, &link.workdir_id)
let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id)
else {
return false;
};
@@ -533,9 +585,8 @@ fn project_working_directory(
}
worker.workdir_attachments = attachments;
} else {
let Ok(has_link_history) = api
.store
.worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref)
let Ok(has_link_history) =
store.worker_workdir_link_history_exists(workspace_id, worker_ref)
else {
return false;
};
@@ -543,13 +594,17 @@ fn project_working_directory(
worker.workdir_attachments.clear();
}
}
project_repository_keys(api, worker)
project_repository_keys(store, workspace_id, worker)
}
fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool {
fn project_repository_keys(
store: &dyn crate::store::ControlPlaneStore,
workspace_id: &str,
worker: &mut SubscriptionWorker,
) -> bool {
project_repository_keys_with(worker, |repository_id| {
api.store
.get_repository(&api.config.workspace_id, repository_id)
store
.get_repository(workspace_id, repository_id)
.ok()
.flatten()
.map(|repository| repository.repository_key)
@@ -634,9 +689,8 @@ mod tests {
protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap()
}
#[test]
fn external_workdir_attachment_does_not_remove_workspace_worker() {
let mut worker = SubscriptionWorker {
fn external_worker() -> SubscriptionWorker {
SubscriptionWorker {
restore_observation_token: None,
worker_id: worker_id(),
runtime_id: Some("runtime-1".to_string()),
@@ -660,8 +714,12 @@ mod tests {
.unwrap(),
},
],
};
}
}
#[test]
fn external_workdir_attachment_does_not_remove_workspace_worker() {
let mut worker = external_worker();
assert!(project_repository_keys_with(&mut worker, |_| {
panic!("External Workdir must not require Repository projection")
}));
@@ -673,6 +731,109 @@ mod tests {
);
}
#[tokio::test(flavor = "current_thread")]
async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository()
{
use crate::store::{
AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord,
WorkspaceRecord,
};
let store =
std::sync::Arc::new(SqliteWorkspaceStore::in_memory().unwrap().with_db_executor(
crate::db_executor::DbExecutor::new(Default::default()).unwrap(),
));
store
.upsert_account(&AccountRecord {
account_id: "owner".into(),
kind: "user".into(),
handle: "owner".into(),
display_name: "Owner".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace-1".into(),
owner_account_id: "owner".into(),
display_name: "Enrichment".into(),
state: "active".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.await
.unwrap();
let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1");
store
.upsert_worker_registry(&WorkerRegistryRecord {
workspace_id: "workspace-1".into(),
worker: worker_ref.clone(),
display_name: "Durable name".into(),
profile: Some("builtin:coder".into()),
retention_state: "normal".into(),
transcript_ref: None,
session_ref: None,
summary_ref: None,
diagnostics_ref: None,
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2")
.unwrap();
let committed = store
.worker_registry_projection("workspace-1", &worker_ref)
.unwrap()
.unwrap();
// Change authority after capturing the payload, as can happen while async
// enrichment is queued. It must not re-query and replace the committed Worker.
let mut later = external_worker();
later.state = protocol::subscription::SubscriptionWorkerState::Running;
store
.apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3")
.unwrap();
let projected = project_registry_workers_async(
store.clone(),
"workspace-1".into(),
vec![committed.clone()],
)
.await
.unwrap();
assert_eq!(projected.len(), 1);
let worker = &projected[0];
assert_eq!(
worker.state,
protocol::subscription::SubscriptionWorkerState::Idle
);
assert_eq!(
worker.availability,
protocol::subscription::SubscriptionWorkerAvailability::Observed
);
assert_eq!(worker.display_name.as_deref(), Some("Durable name"));
assert_eq!(worker.profile.as_deref(), Some("builtin:coder"));
assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1"));
assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1"));
assert_eq!(
worker.workdir_attachments,
external_worker().workdir_attachments
);
let mut unresolved = committed;
unresolved
.observation
.as_mut()
.unwrap()
.worker
.workdir_attachments[0]
.repository_id = Some("missing-repository".into());
assert!(
project_registry_workers_async(store, "workspace-1".into(), vec![unresolved])
.await
.unwrap()
.is_empty()
);
}
async fn run_with_events(events: mpsc::Receiver<protocol::Event>) -> SubscriptionResponse {
let (control_outbound, mut control_receiver) = mpsc::channel(1);
let (protocol_outbound, _protocol_receiver) = mpsc::channel(1);
+95
View File
@@ -0,0 +1,95 @@
# Backend bounded synchronous execution
## Ownership and admission
`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner.
Use `executor.run(move || synchronous_operation(...)).await` with owned input and
services. The closure owns the **whole** synchronous operation, including connection
acquisition, mutex wait, the existing SQL transaction, commit, and ordered local
publication. Do not split source authorization, receipts, CAS, audit, or saved
results around an await. Do not change connection ownership or introduce a read pool.
Production stores share one DB worker and a queue of 32 jobs. Admission uses a
nonblocking queue send and a separate actual-job budget (worker count plus queue
capacity); rejection means the operation was **not accepted**. HTTP maps capacity
and executor unavailability to 503, not authentication failure or deletion conflict.
Tests can inject a dedicated executor with explicit limits, without process-global
settings or environment variables.
Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter:
- does not interrupt, resubmit, or retry SQL;
- does not return its queue/job slot before execution and discarded-result cleanup;
- does not release the request lifecycle lease before actual completion.
Synchronous post-commit continuations inherit the admitted lifecycle source when
submitting another bounded job. This is not implicit inheritance by detached Tokio
tasks. An operation panic has an unknown outcome even if SQL already committed;
return a structured unknown-outcome error and diagnostic, never retry the mutation.
A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all
jobs accepted before its barrier is first polled and waits for actual cleanup; it
is not a queue close and does not include later admissions. Lifecycle callers must
stop their producers/admission before destroying resources.
## Runtime/network boundaries
A synchronous Runtime workflow must not monopolize the sole DB worker while
waiting for callbacks that themselves need DB authority. The Server's separate
finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job
ownership mechanism, not an unbounded `spawn_blocking` queue.
Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can
use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an
owned workflow needs async resource locks, `run_host_workflow` polls it on a Host
worker using the original Tokio handle; do not build a replacement runtime for
cached services and do not submit to and await the Host lane from inside itself.
Control/session guards needed by an accepted mutation belong to that actual job.
Backend Job cleanup is admitted before its durable cleanup claim. Rejection leaves
Pending/Failed ownership recoverable instead of abandoning a Running claim. Ticket
checker dispatch and timeout processing use bounded admission; overload leaves the
saved reservation/deadline diagnosable. They are not automatically retried after an
unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof
of model processing.
Remote subscription and Workdir authorization are async callbacks: their complete
live binding check and fresh request signing are offloaded together. A queue error
must not fall back to a configured bearer token. Synchronous CLI/migration callers
and existing synchronous Runtime adapters retain their explicit blocking boundary.
## Caller recipe (including assignment APIs)
```rust
let store = api.store.clone();
let workspace_id = api.workspace_id().to_owned();
let worker = worker.clone();
let record = api.store.db_executor().run(move || {
store.get_worker_registry(&workspace_id, &worker)
}).await?;
```
This is an illustration, not a new authority API. Assignment delivery scan,
reserve, prepare, acknowledgement and start authorization must keep their existing
whole transaction helpers. Additional assignment diagnostics, answer mutations,
and Ticket queries use the same owned caller boundary; do not add a second ledger
or reconstruct processing evidence from Worker liveness.
## Regression evidence
- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread
timer/control task progresses; accepted/cancelled queue bounds; rollback; panic
before/after commit; discarded reply cleanup and retained authority; child jobs.
- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope
opening/mutation under a separate SQLite connection's write lock; overload;
cancelled HTTP jobs and Workspace deletion drain.
- `worker_projection::tests`: real SQL contention, commit/payload publication,
ordered snapshot registration, and replacement draining a cancelled consumer's
accepted SQL before applying a new Runtime snapshot.
- Auth/proof and Runtime subscription tests protect availability classification,
replay rejection, and no bearer fallback.
- `workdir`'s `http-client` tests protect awaited fresh authorization and exact
request claims before any HTTP send.
These fixtures use temporary authority data and observable synchronization, not
live databases or a fixed sleep to guess completion. This change does not claim a
latency benchmark or a measured production incident.