feat(backend): bound synchronous database and host execution ownership
This commit is contained in:
+145
-16
@@ -432,16 +432,23 @@ mod client {
|
||||
use super::*;
|
||||
use crate::{Workdir, WorkdirSession};
|
||||
|
||||
/// Provides a fresh bearer token for each Runtime request. Backend
|
||||
/// implementations can mint short-lived capability tokens without making a
|
||||
/// Worker-bound session expire with the token used to open it.
|
||||
/// An owned, Send authorization future that can offload synchronous authority
|
||||
/// work without borrowing request data or the authorization provider.
|
||||
pub type WorkdirHttpAuthorizationFuture = std::pin::Pin<
|
||||
Box<dyn std::future::Future<Output = Result<String, WorkdirError>> + Send + 'static>,
|
||||
>;
|
||||
|
||||
/// Provides a fresh bearer token asynchronously for each Runtime request.
|
||||
/// Backend implementations can mint short-lived capability tokens without
|
||||
/// making a Worker-bound session expire with the token used to open it.
|
||||
/// Blocking DB/signing work must be offloaded by the implementation.
|
||||
pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: &str,
|
||||
path_and_query: &str,
|
||||
body: &[u8],
|
||||
) -> Result<String, WorkdirError>;
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture;
|
||||
}
|
||||
|
||||
struct FixedBearerToken(Arc<str>);
|
||||
@@ -455,11 +462,12 @@ mod client {
|
||||
impl WorkdirHttpAuthorization for FixedBearerToken {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
_method: &str,
|
||||
_path_and_query: &str,
|
||||
_body: &[u8],
|
||||
) -> Result<String, WorkdirError> {
|
||||
Ok(self.0.to_string())
|
||||
_method: String,
|
||||
_path_and_query: String,
|
||||
_body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let token = self.0.clone();
|
||||
Box::pin(async move { Ok(token.to_string()) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,7 +517,9 @@ mod client {
|
||||
)?;
|
||||
let body = serde_json::to_vec(&request)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
|
||||
let token = authorization.bearer_token("POST", url.path(), &body)?;
|
||||
let token = authorization
|
||||
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
|
||||
.await?;
|
||||
let response = client
|
||||
.post(url)
|
||||
.bearer_auth(token)
|
||||
@@ -560,7 +570,10 @@ mod client {
|
||||
let operation = WorkdirSessionOperationRequest { operation };
|
||||
let body = serde_json::to_vec(&operation)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
|
||||
let token = self.authorization.bearer_token("POST", url.path(), &body)?;
|
||||
let token = self
|
||||
.authorization
|
||||
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
|
||||
.await?;
|
||||
let response = self
|
||||
.client
|
||||
.post(url)
|
||||
@@ -864,7 +877,10 @@ mod client {
|
||||
&self.base_url,
|
||||
&["v1", "workdir-sessions", self.session_id.as_str()],
|
||||
)?;
|
||||
let token = self.authorization.bearer_token("DELETE", url.path(), &[])?;
|
||||
let token = self
|
||||
.authorization
|
||||
.bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new())
|
||||
.await?;
|
||||
let response = self
|
||||
.client
|
||||
.delete(url)
|
||||
@@ -918,11 +934,124 @@ mod client {
|
||||
WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod authorization_tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct RejectAuthorization {
|
||||
requests: Arc<Mutex<Vec<(String, String, Vec<u8>)>>>,
|
||||
}
|
||||
|
||||
impl WorkdirHttpAuthorization for RejectAuthorization {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let requests = self.requests.clone();
|
||||
Box::pin(async move {
|
||||
tokio::task::yield_now().await;
|
||||
let mut requests = requests.lock().unwrap();
|
||||
requests.push((method, path_and_query, body));
|
||||
Err(WorkdirError::Unavailable(format!(
|
||||
"authorization rejected request {}",
|
||||
requests.len()
|
||||
)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let base_url: Url = format!("http://{}", listener.local_addr().unwrap())
|
||||
.parse()
|
||||
.unwrap();
|
||||
let client = Client::builder().no_proxy().build().unwrap();
|
||||
let authorization = Arc::new(RejectAuthorization::default());
|
||||
let workdir = Workdir::new("w");
|
||||
let open_request = OpenWorkdirSessionRequest {
|
||||
owner_worker_id: Some("worker".to_string()),
|
||||
};
|
||||
let error = RemoteWorkdirSession::open_with_authorization(
|
||||
client.clone(),
|
||||
base_url.clone(),
|
||||
authorization.clone(),
|
||||
workdir.id().clone(),
|
||||
open_request.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 1"
|
||||
));
|
||||
|
||||
// Seed an already-open session to exercise the public operation and
|
||||
// close boundaries independently of the rejected open request.
|
||||
let remote = RemoteWorkdirSession {
|
||||
client,
|
||||
base_url,
|
||||
authorization: authorization.clone(),
|
||||
workdir,
|
||||
session_id: WorkdirSessionId::new("s").unwrap(),
|
||||
capabilities: WorkdirSessionCapabilities::EMPTY,
|
||||
closed: AtomicBool::new(false),
|
||||
};
|
||||
let stat = StatRequest {
|
||||
path: crate::WorkdirPath::new("file").unwrap(),
|
||||
};
|
||||
let error = remote.stat(stat.clone()).await.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 2"
|
||||
));
|
||||
let error = remote.close().await.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 3"
|
||||
));
|
||||
assert_eq!(
|
||||
*authorization.requests.lock().unwrap(),
|
||||
vec![
|
||||
(
|
||||
"POST".to_string(),
|
||||
"/v1/working-directories/w/sessions".to_string(),
|
||||
serde_json::to_vec(&open_request).unwrap(),
|
||||
),
|
||||
(
|
||||
"POST".to_string(),
|
||||
"/v1/workdir-sessions/s/operations".to_string(),
|
||||
serde_json::to_vec(&WorkdirSessionOperationRequest {
|
||||
operation: WorkdirSessionOperation::Stat(stat),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
(
|
||||
"DELETE".to_string(),
|
||||
"/v1/workdir-sessions/s".to_string(),
|
||||
Vec::new(),
|
||||
),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
listener.accept().unwrap_err().kind(),
|
||||
std::io::ErrorKind::WouldBlock
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub use self::RemoteWorkdirSession as ClientSession;
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-client")]
|
||||
pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization};
|
||||
pub use client::{
|
||||
ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
@@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option<String> {
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
|
||||
store: &S,
|
||||
pub async fn resolve_request_actor(
|
||||
store: std::sync::Arc<dyn ControlPlaneStore>,
|
||||
headers: &HeaderMap,
|
||||
cookie_name: &str,
|
||||
) -> Result<Option<RequestActor>> {
|
||||
if let Some(token) = parse_bearer(headers) {
|
||||
let bearer = parse_bearer(headers);
|
||||
let session = parse_cookie(headers, cookie_name);
|
||||
if bearer.is_none() && session.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let executor = store.db_executor();
|
||||
executor
|
||||
.run(move || resolve_request_actor_sync(store.as_ref(), bearer, session))
|
||||
.await
|
||||
}
|
||||
|
||||
// One owned synchronous stage includes token usage marking and the user read.
|
||||
// Do not call async store wrappers here: they use the same executor themselves.
|
||||
fn resolve_request_actor_sync(
|
||||
store: &dyn ControlPlaneStore,
|
||||
bearer: Option<String>,
|
||||
session: Option<String>,
|
||||
) -> Result<Option<RequestActor>> {
|
||||
if let Some(token) = bearer {
|
||||
let token_hash = token_hash(&token);
|
||||
if let Some(api_token) = store.resolve_api_token(&token_hash)? {
|
||||
if api_token
|
||||
@@ -109,7 +127,7 @@ pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(session_token) = parse_cookie(headers, cookie_name) {
|
||||
if let Some(session_token) = session {
|
||||
if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? {
|
||||
if is_expired(&session.expires_at) {
|
||||
return Ok(None);
|
||||
@@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::{
|
||||
AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn seeded_store() -> Arc<SqliteWorkspaceStore> {
|
||||
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::in_memory()
|
||||
.unwrap()
|
||||
.with_db_executor(executor),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-auth".into(),
|
||||
kind: "user".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_user(&UserRecord {
|
||||
user_id: "user-auth".into(),
|
||||
account_id: "account-auth".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.create_browser_session(&BrowserSessionRecord {
|
||||
session_id: "session-auth".into(),
|
||||
token_hash: token_hash("session-secret"),
|
||||
user_id: "user-auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: "9999-01-01T00:00:00Z".into(),
|
||||
revoked_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() {
|
||||
let store = seeded_store();
|
||||
let hash = token_hash("api-secret");
|
||||
store
|
||||
.create_api_token(&ApiTokenRecord {
|
||||
token_id: "token-auth".into(),
|
||||
token_hash: hash.clone(),
|
||||
user_id: "user-auth".into(),
|
||||
label: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: None,
|
||||
revoked_at: None,
|
||||
last_used_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap());
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let actor = resolve_request_actor(store.clone(), &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(actor.user_id, "user-auth");
|
||||
assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken);
|
||||
assert!(
|
||||
store
|
||||
.resolve_api_token(&hash)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.last_used_at
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_api_token_does_not_fall_back_to_valid_browser_session() {
|
||||
let store = seeded_store();
|
||||
let hash = token_hash("expired-secret");
|
||||
store
|
||||
.create_api_token(&ApiTokenRecord {
|
||||
token_id: "token-auth".into(),
|
||||
token_hash: hash.clone(),
|
||||
user_id: "user-auth".into(),
|
||||
label: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: Some("2000-01-01T00:00:00Z".into()),
|
||||
revoked_at: None,
|
||||
last_used_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::AUTHORIZATION,
|
||||
"Bearer expired-secret".parse().unwrap(),
|
||||
);
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
assert!(
|
||||
resolve_request_actor(store.clone(), &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.resolve_api_token(&hash)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.last_used_at
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_bearer_can_fall_back_to_valid_browser_session() {
|
||||
let store = seeded_store();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::AUTHORIZATION,
|
||||
"Bearer unknown-secret".parse().unwrap(),
|
||||
);
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let actor = resolve_request_actor(store, &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(actor.user_id, "user-auth");
|
||||
assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() {
|
||||
let store = seeded_store();
|
||||
let executor = store.db_executor();
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker_executor = executor.clone();
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_executor
|
||||
.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
release_rx.recv().unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) }));
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(1),
|
||||
resolve_request_actor(store, &headers, "session"),
|
||||
)
|
||||
.await;
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap().unwrap();
|
||||
queued.await.unwrap();
|
||||
assert!(matches!(
|
||||
result.expect("capacity rejection must not wait for queued SQL"),
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_code_uses_the_uuid_random_tail() {
|
||||
|
||||
@@ -0,0 +1,957 @@
|
||||
//! Bounded execution of whole synchronous DB operations, independent of Tokio.
|
||||
//!
|
||||
//! Accepted jobs belong to the workers, not their async waiters. Cancelling a
|
||||
//! waiter does not cancel or retry SQL, release capacity, or release its lease.
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
future::Future,
|
||||
num::NonZeroUsize,
|
||||
panic::{AssertUnwindSafe, catch_unwind},
|
||||
sync::{Arc, Mutex, OnceLock, mpsc},
|
||||
};
|
||||
use tokio::sync::{Notify, oneshot};
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// The default is one FIFO worker and at most 32 waiting operations.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct DbExecutorLimits {
|
||||
pub workers: NonZeroUsize,
|
||||
pub queued: NonZeroUsize,
|
||||
}
|
||||
|
||||
impl Default for DbExecutorLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(32).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A request can retain its lifecycle authority until actual SQL finishes.
|
||||
/// Retention happens at submission, while the caller's task-local scope exists.
|
||||
pub(crate) trait DbOperationLifetime: Send + Sync {
|
||||
fn retain(&self) -> Box<dyn Send>;
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static OPERATION_LIFETIME: Arc<dyn DbOperationLifetime>;
|
||||
}
|
||||
|
||||
pub(crate) async fn with_operation_lifetime<F: Future>(
|
||||
lifetime: Arc<dyn DbOperationLifetime>,
|
||||
future: F,
|
||||
) -> F::Output {
|
||||
OPERATION_LIFETIME.scope(lifetime, future).await
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Accepted {
|
||||
last_job_id: u64,
|
||||
unfinished: BTreeSet<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Tracking {
|
||||
accepted: Mutex<Accepted>,
|
||||
finished: Notify,
|
||||
}
|
||||
|
||||
struct Completion {
|
||||
tracking: Arc<Tracking>,
|
||||
id: u64,
|
||||
}
|
||||
|
||||
impl Drop for Completion {
|
||||
fn drop(&mut self) {
|
||||
self.tracking
|
||||
.accepted
|
||||
.lock()
|
||||
.unwrap()
|
||||
.unfinished
|
||||
.remove(&self.id);
|
||||
self.tracking.finished.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
struct Job {
|
||||
operation: Box<dyn FnOnce() + Send>,
|
||||
// Field drop order releases authority before completion becomes observable.
|
||||
_lifetime: Option<Box<dyn Send>>,
|
||||
lifetime_source: Option<Arc<dyn DbOperationLifetime>>,
|
||||
_completion: Completion,
|
||||
runtime: Option<tokio::runtime::Handle>,
|
||||
diagnostics: tracing::Dispatch,
|
||||
span: tracing::Span,
|
||||
}
|
||||
|
||||
fn report_unknown(accepted_job_id: u64, phase: &'static str) {
|
||||
tracing::error!(
|
||||
code = "db_operation_outcome_unknown",
|
||||
accepted_job_id,
|
||||
phase,
|
||||
error = %Error::DbOperationOutcomeUnknown,
|
||||
"DB job panicked; outcome unknown, operation will not be retried"
|
||||
);
|
||||
}
|
||||
|
||||
/// Clones share the queue and actual-job budget. Workers retain only the
|
||||
/// receiver, so dropping the last sender closes the queue; accepted jobs finish
|
||||
/// before the dedicated threads exit. No thread is created per store open.
|
||||
#[derive(Clone)]
|
||||
pub struct DbExecutor {
|
||||
sender: Option<mpsc::SyncSender<Job>>,
|
||||
tracking: Arc<Tracking>,
|
||||
max_jobs: usize,
|
||||
}
|
||||
|
||||
impl DbExecutor {
|
||||
pub fn new(limits: DbExecutorLimits) -> Result<Self> {
|
||||
let max_jobs = limits
|
||||
.workers
|
||||
.get()
|
||||
.checked_add(limits.queued.get())
|
||||
.ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?;
|
||||
let (sender, receiver) = mpsc::sync_channel::<Job>(limits.queued.get());
|
||||
let receiver = Arc::new(Mutex::new(receiver));
|
||||
for index in 0..limits.workers.get() {
|
||||
let receiver = receiver.clone();
|
||||
std::thread::Builder::new()
|
||||
.name(format!("workspace-db-{index}"))
|
||||
.spawn(move || {
|
||||
loop {
|
||||
let next = receiver.lock().unwrap().recv();
|
||||
let Ok(job) = next else { break };
|
||||
let Job {
|
||||
operation,
|
||||
_lifetime: lifetime,
|
||||
lifetime_source,
|
||||
_completion: completion,
|
||||
runtime,
|
||||
diagnostics,
|
||||
span,
|
||||
} = job;
|
||||
// Keep diagnostics with their submitting subscriber as
|
||||
// well as the runtime; no process-global override.
|
||||
let _diagnostics = tracing::dispatcher::set_default(&diagnostics);
|
||||
let _span = span.enter();
|
||||
// Preserve the submitting runtime for synchronous Host
|
||||
// post-commit hooks, including reply/lease destructors.
|
||||
// enter() supplies context only: never drive a future
|
||||
// or await/nest an executor submission in a transaction.
|
||||
let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter);
|
||||
// A cancelled waiter's discarded result may itself have
|
||||
// a destructor. Keep both the lease and actual-job slot
|
||||
// until that cleanup finishes, even if it panics.
|
||||
let execute = || {
|
||||
if catch_unwind(AssertUnwindSafe(operation)).is_err() {
|
||||
report_unknown(completion.id, "reply_cleanup");
|
||||
}
|
||||
if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() {
|
||||
report_unknown(completion.id, "lifetime_cleanup");
|
||||
}
|
||||
};
|
||||
// Synchronous post-commit continuations can admit another
|
||||
// bounded job. Transfer the original lifecycle source as
|
||||
// well, so its authority survives the parent job/waiter.
|
||||
// This does not inject context into detached Tokio tasks.
|
||||
if let Some(source) = lifetime_source {
|
||||
OPERATION_LIFETIME.sync_scope(source, execute);
|
||||
} else {
|
||||
execute();
|
||||
}
|
||||
drop(completion);
|
||||
}
|
||||
})
|
||||
.map_err(|_| Error::DbExecutorUnavailable)?;
|
||||
}
|
||||
Ok(Self {
|
||||
sender: Some(sender),
|
||||
tracking: Arc::default(),
|
||||
max_jobs,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn shared() -> Self {
|
||||
static SHARED: OnceLock<DbExecutor> = OnceLock::new();
|
||||
SHARED
|
||||
.get_or_init(|| {
|
||||
Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self {
|
||||
sender: None,
|
||||
tracking: Arc::default(),
|
||||
max_jobs: 0,
|
||||
})
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Submit without waiting for queue capacity. The closure must contain the
|
||||
/// entire synchronous operation/transaction, with no nested async runtime.
|
||||
/// The submitting Tokio handle, when present, is entered for synchronous
|
||||
/// post-commit hooks and worker cleanup; it is never used to block_on SQL.
|
||||
pub async fn run<T, E>(
|
||||
&self,
|
||||
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
|
||||
) -> std::result::Result<T, E>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: From<Error> + Send + 'static,
|
||||
{
|
||||
self.submit(operation)?
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into()))
|
||||
}
|
||||
|
||||
/// Admit a fire-and-forget operation immediately, without spawning a Tokio
|
||||
/// task or waiting for capacity. Err means the operation was not accepted;
|
||||
/// Ok means the worker owns it through execution and cleanup, even after
|
||||
/// the submitting task is cancelled. A panic emits a structured unknown-
|
||||
/// outcome diagnostic, never an automatic retry. drain() tracks these jobs.
|
||||
pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> {
|
||||
self.submit(move || {
|
||||
operation();
|
||||
Ok::<_, Error>(())
|
||||
})
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
/// Shared synchronous admission: neither dropped receivers nor cancelled
|
||||
/// callers remove accepted work from the queue or the actual-job budget.
|
||||
fn submit<T, E>(
|
||||
&self,
|
||||
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
|
||||
) -> std::result::Result<oneshot::Receiver<std::result::Result<T, E>>, E>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: From<Error> + Send + 'static,
|
||||
{
|
||||
let sender = self
|
||||
.sender
|
||||
.as_ref()
|
||||
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
|
||||
let runtime = tokio::runtime::Handle::try_current().ok();
|
||||
let diagnostics = tracing::dispatcher::get_default(Clone::clone);
|
||||
let span = tracing::Span::current();
|
||||
let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok();
|
||||
let lifetime = lifetime_source.as_ref().map(|source| source.retain());
|
||||
let (reply, result) = oneshot::channel();
|
||||
// Serialize admission and acceptance snapshots. Worker completion cannot
|
||||
// race acceptance, and cancelled waiters never alter this actual-job cap.
|
||||
{
|
||||
let mut accepted = self.tracking.accepted.lock().unwrap();
|
||||
if accepted.unfinished.len() >= self.max_jobs {
|
||||
return Err(Error::DbCapacityExceeded.into());
|
||||
}
|
||||
let id = accepted
|
||||
.last_job_id
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
|
||||
// Never reuse the identity of a rejected job: its destructor may run
|
||||
// after another submission has acquired the admission lock.
|
||||
accepted.last_job_id = id;
|
||||
let job = Job {
|
||||
operation: Box::new(move || {
|
||||
let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| {
|
||||
report_unknown(id, "operation");
|
||||
Err(Error::DbOperationOutcomeUnknown.into())
|
||||
});
|
||||
let _ = reply.send(outcome);
|
||||
}),
|
||||
_lifetime: lifetime,
|
||||
lifetime_source,
|
||||
runtime,
|
||||
diagnostics,
|
||||
span,
|
||||
_completion: Completion {
|
||||
tracking: self.tracking.clone(),
|
||||
id,
|
||||
},
|
||||
};
|
||||
match sender.try_send(job) {
|
||||
Ok(()) => {
|
||||
accepted.unfinished.insert(id);
|
||||
}
|
||||
Err(error) => {
|
||||
// The rejected Job's completion also locks tracking; drop it only
|
||||
// after releasing the admission lock.
|
||||
drop(accepted);
|
||||
return Err(match error {
|
||||
mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded,
|
||||
mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Wait for every job accepted before this barrier is first polled (not
|
||||
/// later submissions). This bypasses queue capacity and survives cancelled
|
||||
/// job waiters. It does not close the queue or stop new submissions.
|
||||
pub async fn drain(&self) -> Result<()> {
|
||||
if self.sender.is_none() {
|
||||
return Err(Error::DbExecutorUnavailable);
|
||||
}
|
||||
let through = self.tracking.accepted.lock().unwrap().last_job_id;
|
||||
loop {
|
||||
let notified = self.tracking.finished.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self
|
||||
.tracking
|
||||
.accepted
|
||||
.lock()
|
||||
.unwrap()
|
||||
.unfinished
|
||||
.first()
|
||||
.is_none_or(|id| *id > through)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rusqlite::{Connection, TransactionBehavior, ffi};
|
||||
use std::{ffi::c_void, time::Duration};
|
||||
|
||||
fn executor(queued: usize) -> DbExecutor {
|
||||
DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(queued).unwrap(),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn database() -> (tempfile::TempDir, Connection, Connection) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("jobs.sqlite");
|
||||
let owner = Connection::open(&path).unwrap();
|
||||
owner
|
||||
.execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)")
|
||||
.unwrap();
|
||||
let worker = Connection::open(&path).unwrap();
|
||||
(dir, owner, worker)
|
||||
}
|
||||
|
||||
struct BusyGate {
|
||||
waiting: Option<oneshot::Sender<()>>,
|
||||
release: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
// A fixture-owned callback proves SQLite actually reached its lock wait.
|
||||
// Unlike busy_handler's fn pointer API, the SQLite API carries a per-call
|
||||
// context, so tests need neither process-global state nor guessed sleeps.
|
||||
struct BusyRegistration {
|
||||
db: *mut ffi::sqlite3,
|
||||
_gate: Box<BusyGate>,
|
||||
}
|
||||
|
||||
impl BusyRegistration {
|
||||
fn install(conn: &Connection, gate: BusyGate) -> Self {
|
||||
unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 {
|
||||
// The Box remains registered only while this connection and
|
||||
// BusyRegistration are alive, on this same dedicated worker.
|
||||
let gate = unsafe { &mut *context.cast::<BusyGate>() };
|
||||
if let Some(waiting) = gate.waiting.take() {
|
||||
let _ = waiting.send(());
|
||||
if gate.release.recv().is_ok() {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
let mut gate = Box::new(gate);
|
||||
// SAFETY: conn owns the handle; registration is dropped before conn.
|
||||
let db = unsafe { conn.handle() };
|
||||
let result = unsafe {
|
||||
ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast())
|
||||
};
|
||||
assert_eq!(result, ffi::SQLITE_OK);
|
||||
Self { db, _gate: gate }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BusyRegistration {
|
||||
fn drop(&mut self) {
|
||||
// SAFETY: unregister before releasing the callback's boxed context.
|
||||
unsafe {
|
||||
ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn observed<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::time::timeout(Duration::from_secs(5), future)
|
||||
.await
|
||||
.expect("DB job did not reach the expected observable state")
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() {
|
||||
let (_dir, mut owner, mut worker) = database();
|
||||
let lock = owner
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let executor = executor(1);
|
||||
let (waiting, blocked) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut write = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _busy = BusyRegistration::install(
|
||||
&worker,
|
||||
BusyGate {
|
||||
waiting: Some(waiting),
|
||||
release: permit,
|
||||
},
|
||||
);
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
observed(blocked).await.unwrap();
|
||||
let control = tokio::spawn(async { 42 });
|
||||
assert_eq!(observed(control).await.unwrap(), 42);
|
||||
// The timer itself is the contract under test, not synchronization.
|
||||
observed(tokio::time::sleep(Duration::from_millis(1))).await;
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
lock.rollback().unwrap();
|
||||
release.send(()).unwrap();
|
||||
observed(write).await.unwrap();
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
struct Lease(Arc<std::sync::atomic::AtomicUsize>);
|
||||
struct Retained(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl DbOperationLifetime for Lease {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Box::new(Retained(self.0.clone()))
|
||||
}
|
||||
}
|
||||
impl Drop for Retained {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() {
|
||||
let (_dir, mut owner, mut worker) = database();
|
||||
let lock = owner
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let executor = executor(1);
|
||||
let (waiting, blocked) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut running = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _busy = BusyRegistration::install(
|
||||
&worker,
|
||||
BusyGate {
|
||||
waiting: Some(waiting),
|
||||
release: permit,
|
||||
},
|
||||
);
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut running).is_pending());
|
||||
observed(blocked).await.unwrap();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap();
|
||||
let mut queued = Box::pin(with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<()> {
|
||||
let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
// FIFO is observable in durable SQL, not inferred from waiter order.
|
||||
let first: i64 =
|
||||
tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| {
|
||||
row.get(0)
|
||||
})?;
|
||||
assert_eq!(first, 1);
|
||||
tx.execute("INSERT INTO writes VALUES (2)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
assert!(matches!(
|
||||
executor.run(|| Ok::<_, Error>(())).await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
let mut drained = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
lock.rollback().unwrap();
|
||||
release.send(()).unwrap();
|
||||
observed(running).await.unwrap();
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() {
|
||||
let (dir, owner, mut parent_conn) = database();
|
||||
let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap();
|
||||
let parent = executor(1);
|
||||
let child = executor(1);
|
||||
let child_owner = child.clone();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let source: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let (entered, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let (child_entered, child_running) = oneshot::channel();
|
||||
let (finish_child, child_permit) = mpsc::channel();
|
||||
let mut operation = Box::pin(with_operation_lifetime(
|
||||
source,
|
||||
parent.run(move || -> Result<()> {
|
||||
let _ = entered.send(());
|
||||
let _ = permit.recv();
|
||||
let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
child_owner.dispatch(move || {
|
||||
let tx = child_conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let committed: i64 = tx
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(committed, 1, "child must see its parent's committed SQL");
|
||||
let _ = child_entered.send(());
|
||||
let _ = child_permit.recv();
|
||||
tx.execute("INSERT INTO writes VALUES (2)", []).unwrap();
|
||||
tx.commit().unwrap();
|
||||
})?;
|
||||
Ok(())
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(operation);
|
||||
release.send(()).unwrap();
|
||||
observed(child_running).await.unwrap();
|
||||
observed(parent.drain()).await.unwrap();
|
||||
assert_eq!(
|
||||
count.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"accepted continuation lost authority when its parent finished"
|
||||
);
|
||||
let mut drained = Box::pin(child.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
finish_child.send(()).unwrap();
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let error = executor
|
||||
.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.execute("INSERT INTO writes VALUES (NULL)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::Sqlite(_)));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let error = executor
|
||||
.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
panic!("lost outcome after commit");
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let error = with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
panic!("lost outcome before commit");
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
struct DiscardedReplyHook {
|
||||
expected_runtime: tokio::runtime::Id,
|
||||
lease_count: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Option<oneshot::Sender<(bool, usize)>>,
|
||||
}
|
||||
|
||||
impl Drop for DiscardedReplyHook {
|
||||
fn drop(&mut self) {
|
||||
let runtime =
|
||||
tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context");
|
||||
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
|
||||
let same_runtime = runtime.id() == self.expected_runtime;
|
||||
let dropped = self.dropped.take().unwrap();
|
||||
tokio::spawn(async move {
|
||||
let _ = dropped.send((same_runtime, retained));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop()
|
||||
{
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut blocker).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
let expected_runtime = tokio::runtime::Handle::current().id();
|
||||
let (hook, published) = oneshot::channel();
|
||||
let (finish, finishing) = mpsc::channel();
|
||||
let (dropped, reply_dropped) = oneshot::channel();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let operation_count = count.clone();
|
||||
let mut queued = Box::pin(with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<_> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
// Model synchronous Host hooks: publish only after commit, checking
|
||||
// try_current and spawning without driving any future on this worker.
|
||||
let runtime = tokio::runtime::Handle::try_current()
|
||||
.expect("post-commit hook lost runtime context");
|
||||
assert_eq!(runtime.id(), expected_runtime);
|
||||
tokio::spawn(async move {
|
||||
let _ = hook.send(());
|
||||
});
|
||||
let _ = finishing.recv();
|
||||
Ok(DiscardedReplyHook {
|
||||
expected_runtime,
|
||||
lease_count: operation_count,
|
||||
dropped: Some(dropped),
|
||||
})
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
release.send(()).unwrap();
|
||||
observed(blocker).await.unwrap();
|
||||
observed(published).await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
finish.send(()).unwrap();
|
||||
assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1));
|
||||
observed(drain).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() {
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(2).unwrap(),
|
||||
queued: NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let mut releases = Vec::new();
|
||||
// Observe each worker executing before submitting the next operation,
|
||||
// so queue occupancy and overload are deterministic, not timing guesses.
|
||||
for _ in 0..2 {
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut operation = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(operation);
|
||||
releases.push(release);
|
||||
}
|
||||
let (finished, mut queued_finished) = oneshot::channel();
|
||||
let mut queued = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = finished.send(());
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
for _ in 0..8 {
|
||||
let rejected_runs = rejected_runs.clone();
|
||||
assert!(matches!(
|
||||
executor
|
||||
.run(move || {
|
||||
rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok::<_, Error>(())
|
||||
})
|
||||
.await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
}
|
||||
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert!(matches!(
|
||||
queued_finished.try_recv(),
|
||||
Err(oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
for release in releases {
|
||||
release.send(()).unwrap();
|
||||
}
|
||||
observed(drain).await.unwrap();
|
||||
observed(queued_finished).await.unwrap();
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct UnknownDiagnostic {
|
||||
code: Option<String>,
|
||||
phase: Option<String>,
|
||||
accepted_job_id: Option<u64>,
|
||||
}
|
||||
|
||||
impl tracing::field::Visit for UnknownDiagnostic {
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
match field.name() {
|
||||
"code" => self.code = Some(value.to_owned()),
|
||||
"phase" => self.phase = Some(value.to_owned()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
|
||||
if field.name() == "accepted_job_id" {
|
||||
self.accepted_job_id = Some(value);
|
||||
}
|
||||
}
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct DiagnosticCapture(Arc<Mutex<Vec<UnknownDiagnostic>>>);
|
||||
|
||||
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for DiagnosticCapture {
|
||||
fn on_event(
|
||||
&self,
|
||||
event: &tracing::Event<'_>,
|
||||
_: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let mut diagnostic = UnknownDiagnostic::default();
|
||||
event.record(&mut diagnostic);
|
||||
self.0.lock().unwrap().push(diagnostic);
|
||||
}
|
||||
}
|
||||
|
||||
struct DispatchCleanup {
|
||||
expected_runtime: tokio::runtime::Id,
|
||||
lease_count: Arc<std::sync::atomic::AtomicUsize>,
|
||||
started: Option<oneshot::Sender<(bool, usize)>>,
|
||||
release: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
impl Drop for DispatchCleanup {
|
||||
fn drop(&mut self) {
|
||||
// No panicking assertions during unwind: report observations to the
|
||||
// test, and gate cleanup so drain/lifetime behavior is observable.
|
||||
let same_runtime = tokio::runtime::Handle::try_current()
|
||||
.is_ok_and(|runtime| runtime.id() == self.expected_runtime);
|
||||
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
|
||||
if let Some(started) = self.started.take() {
|
||||
let _ = started.send((same_runtime, retained));
|
||||
}
|
||||
let _ = self.release.recv();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry()
|
||||
{
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut blocker).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(blocker); // Cancellation does not free its running slot.
|
||||
let expected_runtime = tokio::runtime::Handle::current().id();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let operation_count = count.clone();
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let operation_attempts = attempts.clone();
|
||||
let (cleanup_started, cleaning_up) = oneshot::channel();
|
||||
let (finish, finishing) = mpsc::channel();
|
||||
// Model multiple tracing contexts without setting a global subscriber.
|
||||
// This also prevents tracing's single-subscriber callsite optimization
|
||||
// from caching an unsubscribed parallel test's Interest::never here.
|
||||
let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry());
|
||||
let captured = DiagnosticCapture::default();
|
||||
let diagnostics =
|
||||
tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone()));
|
||||
with_operation_lifetime(lease, async {
|
||||
tracing::dispatcher::with_default(&diagnostics, || {
|
||||
executor.dispatch(move || {
|
||||
let _cleanup = DispatchCleanup {
|
||||
expected_runtime,
|
||||
lease_count: operation_count,
|
||||
started: Some(cleanup_started),
|
||||
release: finishing,
|
||||
};
|
||||
operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let tx = worker
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
tx.execute("INSERT INTO writes VALUES (1)", []).unwrap();
|
||||
tx.commit().unwrap();
|
||||
panic!("dispatched post-commit outcome lost");
|
||||
})
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(); // Dispatch is admitted synchronously; no waiter survives.
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let rejected_operation = rejected_runs.clone();
|
||||
assert!(matches!(
|
||||
executor.dispatch(move || {
|
||||
rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}),
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
release.send(()).unwrap();
|
||||
assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1));
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
finish.send(()).unwrap();
|
||||
observed(drain).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
{
|
||||
let events = captured.0.lock().unwrap();
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(
|
||||
events[0].code.as_deref(),
|
||||
Some("db_operation_outcome_unknown")
|
||||
);
|
||||
assert_eq!(events[0].phase.as_deref(), Some("operation"));
|
||||
assert!(events[0].accepted_job_id.is_some());
|
||||
}
|
||||
let (completed, completion) = oneshot::channel();
|
||||
executor
|
||||
.dispatch(move || {
|
||||
let _ = completed.send(());
|
||||
})
|
||||
.unwrap();
|
||||
observed(executor.drain()).await.unwrap();
|
||||
observed(completion).await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let job = store
|
||||
@@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let job = store
|
||||
|
||||
@@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
};
|
||||
let migrated = store
|
||||
@@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let archived = store
|
||||
@@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let service =
|
||||
@@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() {
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let service =
|
||||
|
||||
@@ -533,6 +533,7 @@ mod existing_workspace_provisioning {
|
||||
// This fixture contains only the schema-87 cutover authorities, not the
|
||||
// unrelated full Server schema. Exercise the real service/store methods.
|
||||
Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
})
|
||||
}
|
||||
@@ -644,6 +645,7 @@ mod existing_workspace_provisioning {
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())),
|
||||
});
|
||||
let service = WorkspaceSigningIdentityService::new(
|
||||
|
||||
@@ -23,7 +23,10 @@ use std::{
|
||||
};
|
||||
use workdir::{
|
||||
Workdir, WorkdirError, WorkdirSessionHandle,
|
||||
http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization},
|
||||
http::{
|
||||
OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization,
|
||||
WorkdirHttpAuthorizationFuture,
|
||||
},
|
||||
};
|
||||
use worker_runtime::RuntimeWorkspaceScope;
|
||||
use worker_runtime::catalog::{
|
||||
@@ -3473,6 +3476,38 @@ impl WorkspaceRuntimeAuthorization {
|
||||
}
|
||||
}
|
||||
|
||||
/// Offload the complete live-binding check and request signing operation.
|
||||
/// Claims are minted inside the bounded job, not cached before admission.
|
||||
pub(crate) async fn issue_async(
|
||||
&self,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
operation: String,
|
||||
worker_id: Option<String>,
|
||||
body: Vec<u8>,
|
||||
) -> Result<String, RuntimeDiagnostic> {
|
||||
let executor = self.store.db_executor();
|
||||
let authorization = self.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
Ok::<_, Error>(authorization.issue(
|
||||
&method,
|
||||
&path_and_query,
|
||||
&operation,
|
||||
worker_id.as_deref(),
|
||||
&body,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
diagnostic(
|
||||
"workspace_runtime_authorization_unavailable",
|
||||
HostDiagnosticSeverity::Error,
|
||||
error.to_string(),
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
pub(crate) fn issue(
|
||||
&self,
|
||||
method: &str,
|
||||
@@ -3645,26 +3680,25 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization {
|
||||
impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: &str,
|
||||
path_and_query: &str,
|
||||
body: &[u8],
|
||||
) -> Result<String, WorkdirError> {
|
||||
if let Some(authorization) = &self.workspace_authorization {
|
||||
return authorization
|
||||
.issue(
|
||||
method,
|
||||
path_and_query,
|
||||
workspace_runtime_operation(method, path_and_query),
|
||||
None,
|
||||
body,
|
||||
)
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let authorization = self.clone();
|
||||
Box::pin(async move {
|
||||
if let Some(authorizer) = &authorization.workspace_authorization {
|
||||
let operation = workspace_runtime_operation(&method, &path_and_query).to_string();
|
||||
return authorizer
|
||||
.issue_async(method, path_and_query, operation, None, body)
|
||||
.await
|
||||
.map_err(|error| WorkdirError::Unavailable(error.message));
|
||||
}
|
||||
self.fallback_bearer_token.clone().ok_or_else(|| {
|
||||
authorization.fallback_bearer_token.ok_or_else(|| {
|
||||
WorkdirError::Unavailable(
|
||||
"remote Runtime does not have bearer authorization configured".to_string(),
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6344,6 +6378,79 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() {
|
||||
use crate::db_executor::{DbExecutor, DbExecutorLimits};
|
||||
use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding};
|
||||
use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore;
|
||||
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::in_memory()
|
||||
.unwrap()
|
||||
.with_db_executor(executor.clone()),
|
||||
);
|
||||
let service = WorkspaceSigningIdentityService::new(
|
||||
store.clone(),
|
||||
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
|
||||
);
|
||||
let authorization = RemoteWorkdirAuthorization {
|
||||
workspace_authorization: Some(WorkspaceRuntimeAuthorization::new(
|
||||
store,
|
||||
service,
|
||||
"https://backend.test",
|
||||
WorkspaceRuntimeBinding {
|
||||
workspace_id: "workspace".into(),
|
||||
runtime_id: "runtime".into(),
|
||||
display_name: "Runtime".into(),
|
||||
base_url: "https://runtime.test".into(),
|
||||
public_key_fingerprint: "unused".into(),
|
||||
public_key: "unused".into(),
|
||||
binding_id: "binding".into(),
|
||||
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
revoked_at: None,
|
||||
},
|
||||
)),
|
||||
fallback_bearer_token: Some("must-not-be-used".into()),
|
||||
};
|
||||
|
||||
// Hold the running slot and fill the only queue slot. The callback must
|
||||
// reject admission, not run inline or silently switch authentication.
|
||||
let (release, wait) = std::sync::mpsc::channel::<()>();
|
||||
let (started, running) = tokio::sync::oneshot::channel();
|
||||
executor
|
||||
.dispatch(move || {
|
||||
let _ = started.send(());
|
||||
let _ = wait.recv_timeout(Duration::from_secs(5));
|
||||
})
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), running)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
executor.dispatch(|| {}).unwrap();
|
||||
let error = authorization
|
||||
.bearer_token(
|
||||
"POST".into(),
|
||||
"/v1/workdir-sessions/s/operations".into(),
|
||||
b"{}".to_vec(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string()
|
||||
));
|
||||
drop(release);
|
||||
executor.drain().await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ticket_worker_spawn_intent_uses_worker_wire_role() {
|
||||
let intent = WorkerSpawnIntent::TicketRole {
|
||||
|
||||
@@ -10,6 +10,7 @@ pub mod backend_job;
|
||||
pub mod companion;
|
||||
pub mod config;
|
||||
pub mod config_source;
|
||||
pub mod db_executor;
|
||||
pub mod feature_storage;
|
||||
pub mod hosts;
|
||||
pub mod memory_backend;
|
||||
@@ -159,6 +160,12 @@ pub enum Error {
|
||||
WorkspaceSigningIdentity { code: String, message: String },
|
||||
#[error("Workspace Drive error: {0}")]
|
||||
Drive(#[from] workspace_drive::Error),
|
||||
#[error("DB execution capacity exceeded")]
|
||||
DbCapacityExceeded,
|
||||
#[error("DB executor unavailable")]
|
||||
DbExecutorUnavailable,
|
||||
#[error("DB operation outcome unknown; do not automatically retry")]
|
||||
DbOperationOutcomeUnknown,
|
||||
#[error("store error: {0}")]
|
||||
Store(String),
|
||||
}
|
||||
|
||||
@@ -965,7 +965,7 @@ async fn connect_runtime(
|
||||
let mut request = endpoint
|
||||
.into_client_request()
|
||||
.map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?;
|
||||
if let Some(token) = runtime_token(config, workspace_id)? {
|
||||
if let Some(token) = runtime_token(config, workspace_id).await? {
|
||||
request.headers_mut().insert(
|
||||
"authorization",
|
||||
format!("Bearer {token}")
|
||||
@@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String {
|
||||
format!("{base}/v1/protocol/ws")
|
||||
}
|
||||
}
|
||||
fn runtime_token(
|
||||
async fn runtime_token(
|
||||
config: &RemoteRuntimeConfig,
|
||||
_workspace_id: &str,
|
||||
) -> Result<Option<String>, String> {
|
||||
if let Some(authorization) = config.workspace_authorization.as_ref() {
|
||||
return authorization
|
||||
.issue("GET", "/v1/protocol/ws", "workers:list", None, &[])
|
||||
.issue_async(
|
||||
"GET".to_string(),
|
||||
"/v1/protocol/ws".to_string(),
|
||||
"workers:list".to_string(),
|
||||
None,
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(|error| error.message);
|
||||
}
|
||||
|
||||
+2305
-703
File diff suppressed because it is too large
Load Diff
@@ -65,12 +65,18 @@ pub(super) fn require_candidate_grant(
|
||||
/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface
|
||||
/// background work; the new Job regenerates it and does not infer surface success.
|
||||
pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult<bool> {
|
||||
let job_api = api.clone();
|
||||
let subject_id = subject_id.to_owned();
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let (worker, complete) = host_io_executor()?
|
||||
.run(move || {
|
||||
let api = &job_api;
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let Some(owner) = api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
else {
|
||||
return Ok(true);
|
||||
return Ok((None, true));
|
||||
};
|
||||
let mut summary = api
|
||||
.runtime
|
||||
@@ -85,7 +91,7 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
|
||||
if !stopped && !idle {
|
||||
return Ok(false);
|
||||
return Ok((None, false));
|
||||
}
|
||||
if api
|
||||
.store
|
||||
@@ -106,14 +112,15 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
|
||||
&owner.worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(
|
||||
"Retire legacy subjektiv consolidator after foreground completion".into(),
|
||||
"Retire legacy subjektiv consolidator after foreground completion"
|
||||
.into(),
|
||||
),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(false);
|
||||
return Ok((None, false));
|
||||
}
|
||||
}
|
||||
if !api
|
||||
@@ -123,12 +130,18 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
|
||||
.state
|
||||
.eq_ignore_ascii_case("stopped")
|
||||
{
|
||||
return Ok(false);
|
||||
return Ok((None, false));
|
||||
}
|
||||
Ok::<_, ApiError>((Some(owner.worker), false))
|
||||
})
|
||||
.await?;
|
||||
let Some(worker) = worker else {
|
||||
return Ok(complete);
|
||||
};
|
||||
let removal = WorkerRemovalService::new(api)
|
||||
.execute_target_removal(
|
||||
api.runtime.as_ref(),
|
||||
&owner.worker,
|
||||
&worker,
|
||||
"subjektiv consolidation Job cutover",
|
||||
None,
|
||||
)
|
||||
@@ -141,10 +154,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
|
||||
))
|
||||
.into());
|
||||
}
|
||||
Ok(api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.is_none())
|
||||
let store = api.store.clone();
|
||||
let workspace_id = api.workspace_id().to_owned();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store
|
||||
.current_worker_singleton_owner(&workspace_id, &key)
|
||||
.map(|owner| owner.is_none())
|
||||
.map_err(ApiError::from)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn bounded_candidate_batch(
|
||||
|
||||
@@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m
|
||||
&error,
|
||||
)
|
||||
});
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
|
||||
@@ -767,6 +767,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) {
|
||||
std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 })
|
||||
);
|
||||
assert!(execution.protocol_methods().is_empty());
|
||||
// SQLite jobs run on external threads; paused Tokio time cannot observe their
|
||||
// progress and may auto-advance a reconnect timeout before SQL completes.
|
||||
tokio::time::resume();
|
||||
assert_same_worker_reconnect_and_stop(&worker, &context).await;
|
||||
}
|
||||
|
||||
@@ -844,7 +847,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[tokio::test]
|
||||
async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress()
|
||||
{
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
|
||||
let f = Fixture::new().await;
|
||||
let mut a = f.attach("1");
|
||||
let (first, reply) = f.start_read("1", &mut a).await;
|
||||
let mut second = Box::pin(f.app.clone().oneshot(f.operation("1")));
|
||||
// Poll through dispatch/auth until the handler waits on its resource lock.
|
||||
let request = f.operation("1");
|
||||
let proof = request
|
||||
.headers()
|
||||
.get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER)
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap();
|
||||
let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof)
|
||||
.unwrap()
|
||||
.jti;
|
||||
let mut second = Box::pin(f.app.clone().oneshot(request));
|
||||
// DB auth is now genuinely asynchronous. Observe durable proof consumption,
|
||||
// not a guessed number of polls, before advancing the injected proof clock.
|
||||
let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
tokio::time::timeout(BOUND, async {
|
||||
loop {
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let consumed: bool = connection
|
||||
.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)",
|
||||
[&jti],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
if consumed {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second request never authenticated before resource wait");
|
||||
assert!(
|
||||
a.try_recv().is_err(),
|
||||
"same session read escaped resource exclusion"
|
||||
@@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
|
||||
assert_json(first.await.unwrap(), StatusCode::OK).await;
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let Some(ExternalProviderCommand::Operation { response, .. }) =
|
||||
tokio::time::timeout(BOUND, a.recv()).await.unwrap()
|
||||
tokio::time::timeout(BOUND, async {
|
||||
tokio::select! {
|
||||
command = a.recv() => command,
|
||||
result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()),
|
||||
}
|
||||
}).await.unwrap()
|
||||
else {
|
||||
panic!("expected second read")
|
||||
};
|
||||
@@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits(
|
||||
}),
|
||||
);
|
||||
let mut deletion = Box::pin(f.app.clone().oneshot(request));
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await;
|
||||
assert!(
|
||||
gate.admit(false).is_none(),
|
||||
"deletion start did not fence admission"
|
||||
);
|
||||
tokio::time::timeout(BOUND, async {
|
||||
loop {
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
if gate.admit(false).is_none() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("deletion start did not fence admission after async authentication");
|
||||
let session_id = session_store::new_session_id().to_string();
|
||||
let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions");
|
||||
let callback = f.request(
|
||||
|
||||
@@ -57,6 +57,19 @@ impl WorkerOperationContext {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) async fn from_request_async(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
) -> std::result::Result<Self, server_api::RepositoryApiError> {
|
||||
let owned_api = api.clone();
|
||||
let context = context.clone();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context)))
|
||||
.await
|
||||
.map_err(|error| ApiError::from(error).into_repository_api_error())?
|
||||
}
|
||||
|
||||
pub(super) fn from_request(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
@@ -162,8 +175,20 @@ impl<Services> WorkspaceWorker<Services> {
|
||||
source: crate::worker_source::VerifiedWorkerMutationSource,
|
||||
reason: &str,
|
||||
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
||||
self.current_record()
|
||||
.map_err(|error| error.error.to_string())?;
|
||||
let removal = self.removal.clone();
|
||||
let identity = self.identity.clone();
|
||||
self.removal
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
removal
|
||||
.store
|
||||
.get_worker_registry(&removal.workspace_id, &identity)?
|
||||
.ok_or_else(|| Error::UnknownWorker { worker: identity })
|
||||
.map(drop)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
self.removal
|
||||
.execute_async(
|
||||
source,
|
||||
@@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor
|
||||
}
|
||||
|
||||
impl WorkspaceWorker {
|
||||
pub(crate) async fn resolve_async(
|
||||
api: &WorkspaceApi,
|
||||
runtime_id: &str,
|
||||
reference: &str,
|
||||
) -> Result<Self> {
|
||||
let owned_api = api.clone();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let reference = reference.to_owned();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || Self::resolve(&owned_api, &runtime_id, &reference))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result<Self> {
|
||||
Self::resolve_with_services(
|
||||
api.clone(),
|
||||
@@ -239,7 +278,36 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
permission: &str,
|
||||
) -> ApiResult<Option<tokio::sync::OwnedMutexGuard<()>>> {
|
||||
self.current_record()?;
|
||||
let worker = self.clone();
|
||||
let initial_context = context.clone();
|
||||
let permission = permission.to_owned();
|
||||
let initial_permission = permission.clone();
|
||||
let grant = self
|
||||
.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<Option<WorkerControlGrantRecord>> {
|
||||
worker.current_record()?;
|
||||
let WorkerOperationContext::WorkerControl { controller } = initial_context else {
|
||||
return Ok(None);
|
||||
};
|
||||
if worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(worker.api.workspace_id(), &controller)?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::UnknownWorker { worker: controller }.into());
|
||||
}
|
||||
Ok(Some(authorize_known_worker_permission(
|
||||
&worker.api,
|
||||
worker.api.workspace_id(),
|
||||
&controller,
|
||||
&worker.identity,
|
||||
&initial_permission,
|
||||
)?))
|
||||
})
|
||||
.await?;
|
||||
match context {
|
||||
#[cfg(test)]
|
||||
WorkerOperationContext::Backend => Ok(None),
|
||||
@@ -248,34 +316,25 @@ impl WorkspaceWorker {
|
||||
Ok(None)
|
||||
}
|
||||
WorkerOperationContext::WorkerControl { controller } => {
|
||||
if self
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(self.api.workspace_id(), controller)?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::UnknownWorker {
|
||||
worker: controller.clone(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let grant = authorize_known_worker_permission(
|
||||
&self.api,
|
||||
self.api.workspace_id(),
|
||||
controller,
|
||||
&self.identity,
|
||||
permission,
|
||||
)?;
|
||||
let grant = grant.expect("Worker control context resolved a grant");
|
||||
// Await admission outside the DB lane; the recheck owns the guard
|
||||
// until it finishes, even if its async waiter is cancelled.
|
||||
let guard = worker_control_lock(&self.api, &grant.grant_id)
|
||||
.lock_owned()
|
||||
.await;
|
||||
self.current_record()?;
|
||||
let worker = self.clone();
|
||||
let controller = controller.clone();
|
||||
self.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<_> {
|
||||
worker.current_record()?;
|
||||
let current_grant = authorize_known_worker_permission(
|
||||
&self.api,
|
||||
self.api.workspace_id(),
|
||||
controller,
|
||||
&self.identity,
|
||||
permission,
|
||||
&worker.api,
|
||||
worker.api.workspace_id(),
|
||||
&controller,
|
||||
&worker.identity,
|
||||
&permission,
|
||||
)?;
|
||||
if current_grant.grant_id != grant.grant_id {
|
||||
return Err(Error::RepositoryConflict(
|
||||
@@ -284,6 +343,8 @@ impl WorkspaceWorker {
|
||||
.into());
|
||||
}
|
||||
Ok(Some(guard))
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -293,8 +354,14 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
) -> ApiResult<InternalWorkerRestoreResult> {
|
||||
let _guard = self.authorize_operation(context, "restore").await?;
|
||||
self.restore_internal(request)
|
||||
let guard = self.authorize_operation(context, "restore").await?;
|
||||
let worker = self.clone();
|
||||
host_io_executor()?
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
worker.restore_internal(request)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Trusted Backend callers still supply an explicit observed intent. Runtime
|
||||
@@ -312,27 +379,43 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: WorkerLifecycleRequest,
|
||||
) -> ApiResult<WorkerLifecycleResult> {
|
||||
let _guard = self.authorize_operation(context, "stop").await?;
|
||||
let result = self
|
||||
let guard = self.authorize_operation(context, "stop").await?;
|
||||
let worker = self.clone();
|
||||
let (result, guard) = host_io_executor()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let result = worker
|
||||
.api
|
||||
.runtime
|
||||
.stop_worker(&self.identity, request)
|
||||
.stop_worker(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?;
|
||||
Ok((result, guard))
|
||||
})
|
||||
.await?;
|
||||
if result.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(result);
|
||||
}
|
||||
parse_runtime_worker_id_for_registry(&self.identity.worker_id)?;
|
||||
let session_lock = current_worker_session_lock(&self.api, &self.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
let session_guard = session_lock.lock_owned().await;
|
||||
close_current_worker_session_locked(&self.api, &self.identity).await?;
|
||||
if let Some(record) = self
|
||||
let worker = self.clone();
|
||||
host_io_executor()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let (_control, _session) = (guard, session_guard);
|
||||
if let Some(record) = worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(self.api.workspace_id(), &self.identity)?
|
||||
.get_worker_registry(worker.api.workspace_id(), &worker.identity)?
|
||||
{
|
||||
sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?;
|
||||
sync_linked_workdir_after_worker_stop(
|
||||
&worker.api,
|
||||
&worker.identity.runtime_id,
|
||||
&record,
|
||||
)?;
|
||||
}
|
||||
Ok(result)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn cancel(
|
||||
@@ -340,12 +423,18 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: WorkerLifecycleRequest,
|
||||
) -> ApiResult<WorkerLifecycleResult> {
|
||||
let _guard = self.authorize_operation(context, "cancel").await?;
|
||||
Ok(self
|
||||
let guard = self.authorize_operation(context, "cancel").await?;
|
||||
let worker = self.clone();
|
||||
host_io_executor()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let _guard = guard;
|
||||
Ok(worker
|
||||
.api
|
||||
.runtime
|
||||
.cancel_worker(&self.identity, request)
|
||||
.cancel_worker(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn input(
|
||||
@@ -358,29 +447,40 @@ impl WorkspaceWorker {
|
||||
} else {
|
||||
"send_input"
|
||||
};
|
||||
let _guard = self.authorize_operation(context, permission).await?;
|
||||
let guard = self.authorize_operation(context, permission).await?;
|
||||
let session_lock = current_worker_session_lock(&self.api, &self.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
self.ensure_interactive()?;
|
||||
let session_guard = session_lock.lock_owned().await;
|
||||
self.ensure_interactive().await?;
|
||||
if matches!(context, WorkerOperationContext::WorkerControl { .. })
|
||||
&& request.kind == WorkerInputKind::User
|
||||
{
|
||||
request.kind = WorkerInputKind::UserIfIdle;
|
||||
}
|
||||
Ok(self
|
||||
let worker = self.clone();
|
||||
host_io_executor()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let (_control, _session) = (guard, session_guard);
|
||||
Ok(worker
|
||||
.api
|
||||
.runtime
|
||||
.send_input(&self.identity, request)
|
||||
.send_input(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn ensure_interactive(&self) -> ApiResult<()> {
|
||||
if let Some(claim) = self
|
||||
async fn ensure_interactive(&self) -> ApiResult<()> {
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<()> {
|
||||
if let Some(claim) = worker
|
||||
.api
|
||||
.store
|
||||
.get_pending_ticket_assignment_operation_for_worker(
|
||||
self.api.workspace_id(),
|
||||
&self.identity,
|
||||
worker.api.workspace_id(),
|
||||
&worker.identity,
|
||||
)?
|
||||
{
|
||||
return Err(Error::TicketAssignmentConflict(format!(
|
||||
@@ -388,16 +488,18 @@ impl WorkspaceWorker {
|
||||
claim.operation_id,
|
||||
)).into());
|
||||
}
|
||||
if self
|
||||
if worker
|
||||
.api
|
||||
.store
|
||||
.worker_registry_projection(self.api.workspace_id(), &self.identity)?
|
||||
.worker_registry_projection(worker.api.workspace_id(), &worker.identity)?
|
||||
.and_then(|projection| projection.job)
|
||||
.is_some()
|
||||
{
|
||||
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn set_pinned(
|
||||
@@ -405,14 +507,23 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
pinned: bool,
|
||||
) -> ApiResult<WorkerRetentionResponse> {
|
||||
let _guard = self.authorize_operation(context, "pin").await?;
|
||||
let retention_state = if pinned { "pinned" } else { "normal" };
|
||||
let changed = self.api.worker_projection.publish_ordered(|| {
|
||||
let guard = self.authorize_operation(context, "pin").await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<WorkerRetentionResponse> {
|
||||
// Publication and its DB mutation are one owned stage. Accepted
|
||||
// work retains control admission through commit and publication.
|
||||
let _guard = guard;
|
||||
let retention_state = if pinned { "pinned" } else { "normal" };
|
||||
let changed = worker.api.worker_projection.publish_ordered(|| {
|
||||
worker
|
||||
.api
|
||||
.store
|
||||
.update_worker_retention(
|
||||
self.api.workspace_id(),
|
||||
&self.identity,
|
||||
worker.api.workspace_id(),
|
||||
&worker.identity,
|
||||
retention_state,
|
||||
now_registry_timestamp().as_str(),
|
||||
)
|
||||
@@ -420,18 +531,20 @@ impl WorkspaceWorker {
|
||||
})?;
|
||||
if !changed {
|
||||
return Err(cleanup_api_error(
|
||||
&self.identity.runtime_id,
|
||||
&worker.identity.runtime_id,
|
||||
"workspace_worker_retention_unknown_worker",
|
||||
"Worker is not known to the Backend registry",
|
||||
));
|
||||
}
|
||||
Ok(WorkerRetentionResponse {
|
||||
workspace_id: self.api.workspace_id().to_owned(),
|
||||
runtime_id: self.identity.runtime_id.clone(),
|
||||
worker_id: self.identity.worker_id.clone(),
|
||||
workspace_id: worker.api.workspace_id().to_owned(),
|
||||
runtime_id: worker.identity.runtime_id.clone(),
|
||||
worker_id: worker.identity.worker_id.clone(),
|
||||
pinned,
|
||||
retention_state: retention_state.into(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// The caller owns the full cleanup-plan digest check. The removal service
|
||||
@@ -611,7 +724,7 @@ impl WorkspaceWorkerMethodSender {
|
||||
.await?;
|
||||
let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
self.worker.ensure_interactive()?;
|
||||
self.worker.ensure_interactive().await?;
|
||||
let WorkerOperationContext::Browser { source, .. } = context else {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"Browser Worker protocol context required".into(),
|
||||
|
||||
@@ -72,6 +72,28 @@ pub(super) struct RequestLease {
|
||||
callback: bool,
|
||||
}
|
||||
|
||||
impl crate::db_executor::DbOperationLifetime for RequestLease {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
// This is continuation of an already admitted request, not fresh admission.
|
||||
// Retain its drain responsibility before transferring work to the DB owner;
|
||||
// dropping/aborting the HTTP waiter must not allow deletion past live SQL.
|
||||
let mut state = self
|
||||
.gate
|
||||
.state
|
||||
.lock()
|
||||
.expect("Workspace admission poisoned");
|
||||
if self.callback {
|
||||
state.callbacks += 1;
|
||||
} else {
|
||||
state.requests += 1;
|
||||
}
|
||||
Box::new(RequestLease {
|
||||
gate: self.gate.clone(),
|
||||
callback: self.callback,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RequestLease {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self
|
||||
|
||||
@@ -0,0 +1,606 @@
|
||||
//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs).
|
||||
use super::*;
|
||||
use crate::SqliteWorkspaceStore;
|
||||
use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime};
|
||||
use rusqlite::{Connection, ffi};
|
||||
use std::{
|
||||
ffi::c_void,
|
||||
future::Future,
|
||||
num::NonZeroUsize,
|
||||
sync::{atomic::AtomicBool, mpsc},
|
||||
thread::JoinHandle,
|
||||
time::Duration as StdDuration,
|
||||
};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
const WORKSPACE: &str = "db-execution";
|
||||
const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects";
|
||||
const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2);
|
||||
const WATCHDOG: StdDuration = StdDuration::from_secs(5);
|
||||
|
||||
async fn observed<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::time::timeout(OBSERVE_LIMIT, future)
|
||||
.await
|
||||
.expect("request did not reach the expected DB/lifecycle state")
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
// Keep temporary authority files alive until all other fixture owners drop.
|
||||
server: WorkspaceServerApi,
|
||||
api: WorkspaceApi,
|
||||
executor: DbExecutor,
|
||||
_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
async fn new() -> Self {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
// Explicit startup config: never resolve the developer's live data dir.
|
||||
let config = ServerConfig {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
workspace_display_name: "DB execution fixture".into(),
|
||||
workspace_created_at: "2026-01-01T00:00:00Z".into(),
|
||||
workspace_execution_root: dir.path().into(),
|
||||
database_path: dir.path().join("server.db"),
|
||||
data_root: dir.path().join("data"),
|
||||
embedded_runtime_store_root: dir.path().join("runtime"),
|
||||
static_assets_dir: None,
|
||||
auth: AuthConfig::Passkey {
|
||||
rp_id: "localhost".into(),
|
||||
origin: "http://localhost:5173".into(),
|
||||
public_base_url: "http://localhost:5173".into(),
|
||||
cookie_name: "db_execution_session".into(),
|
||||
},
|
||||
max_records: 100,
|
||||
repositories: Vec::new(),
|
||||
runtime_event_sources: Vec::new(),
|
||||
remote_runtime_sources: Vec::new(),
|
||||
backend_base_url: Some("http://localhost:8787".into()),
|
||||
};
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open(&config.database_path)
|
||||
.unwrap()
|
||||
.with_db_executor(executor.clone()),
|
||||
);
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_user(&UserRecord {
|
||||
user_id: "fixture-user".into(),
|
||||
account_id: format!("account-{WORKSPACE}"),
|
||||
handle: "fixture-user".into(),
|
||||
display_name: "Fixture user".into(),
|
||||
created_at: config.workspace_created_at.clone(),
|
||||
updated_at: config.workspace_created_at.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.create_browser_session(&crate::store::BrowserSessionRecord {
|
||||
session_id: "fixture-session".into(),
|
||||
token_hash: crate::auth::token_hash("fixture-browser-token"),
|
||||
user_id: "fixture-user".into(),
|
||||
created_at: config.workspace_created_at.clone(),
|
||||
expires_at: "2999-01-01T00:00:00Z".into(),
|
||||
revoked_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let api = WorkspaceApi::new(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let server = WorkspaceServerApi::new(config, store);
|
||||
// Cache only the real inner router; avoid starting the unrelated hook.
|
||||
server
|
||||
.apis
|
||||
.lock()
|
||||
.await
|
||||
.insert(WORKSPACE.into(), api.clone());
|
||||
server
|
||||
.routers
|
||||
.lock()
|
||||
.await
|
||||
.insert(WORKSPACE.into(), build_inner_router(api.clone()));
|
||||
executor.drain().await.unwrap();
|
||||
Self {
|
||||
server,
|
||||
api,
|
||||
executor,
|
||||
_dir: dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn request(method: &str, path: &str, body: serde_json::Value) -> Request {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(path)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header("cookie", "db_execution_session=fixture-browser-token")
|
||||
.header(ORIGIN, "http://localhost:5173")
|
||||
.body(axum::body::Body::from(body.to_string()))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn subject_request(role: &str) -> Request {
|
||||
Self::request(
|
||||
"POST",
|
||||
SUBJECTS,
|
||||
serde_json::json!({ "role": role, "behavior_md": "" }),
|
||||
)
|
||||
}
|
||||
|
||||
fn dispatch(&self, request: Request) -> tokio::task::JoinHandle<Response> {
|
||||
let server = self.server.clone();
|
||||
tokio::spawn(async move { dispatch_workspace_request(State(server), request).await })
|
||||
}
|
||||
|
||||
// A pre-authenticated generated router isolates handler admission from the
|
||||
// earlier browser-session DB read. Full authentication/dispatch is exercised
|
||||
// separately by the cancellation and warm-Feature tests below.
|
||||
fn inner_router(&self) -> Router {
|
||||
build_inner_router(self.api.clone()).layer(Extension(RequestActor {
|
||||
user_id: "fixture-user".into(),
|
||||
account_id: format!("account-{WORKSPACE}"),
|
||||
handle: "fixture-user".into(),
|
||||
display_name: "Fixture user".into(),
|
||||
auth_method: ActorAuthMethod::BrowserSession,
|
||||
}))
|
||||
}
|
||||
|
||||
fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) {
|
||||
let database = self
|
||||
.api
|
||||
.feature_storage
|
||||
.open(&self.api.subjektiv_registration)
|
||||
.unwrap();
|
||||
// Initialize the trusted scope, then obtain SQLite's actual file path
|
||||
// from the managed handle rather than guessing the Feature directory.
|
||||
crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap();
|
||||
let path = database
|
||||
.with_connection(|conn| {
|
||||
Ok(PathBuf::from(conn.query_row(
|
||||
"SELECT file FROM pragma_database_list WHERE name = 'main'",
|
||||
[],
|
||||
|row| row.get::<_, String>(0),
|
||||
)?))
|
||||
})
|
||||
.unwrap();
|
||||
(database, path)
|
||||
}
|
||||
}
|
||||
|
||||
/// SQLite write ownership lives on an OS thread, including a bounded watchdog.
|
||||
/// Drop releases and joins it even when an assertion panics on the async side.
|
||||
struct WriteLock {
|
||||
release: Option<mpsc::Sender<()>>,
|
||||
thread: Option<JoinHandle<()>>,
|
||||
released: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WriteLock {
|
||||
fn acquire(path: PathBuf) -> Self {
|
||||
let (ready, held) = mpsc::sync_channel(1);
|
||||
let (release, permit) = mpsc::channel();
|
||||
let released = Arc::new(AtomicBool::new(false));
|
||||
let flag = released.clone();
|
||||
let thread = std::thread::spawn(move || {
|
||||
let conn = Connection::open(path).unwrap();
|
||||
conn.execute_batch("BEGIN IMMEDIATE").unwrap();
|
||||
ready.send(()).unwrap();
|
||||
let _ = permit.recv_timeout(WATCHDOG);
|
||||
conn.execute_batch("ROLLBACK").unwrap();
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
held.recv_timeout(WATCHDOG)
|
||||
.expect("SQLite lock holder did not acquire its transaction");
|
||||
Self {
|
||||
release: Some(release),
|
||||
thread: Some(thread),
|
||||
released,
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&mut self) {
|
||||
if let Some(release) = self.release.take() {
|
||||
let _ = release.send(());
|
||||
}
|
||||
if let Some(thread) = self.thread.take() {
|
||||
thread.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Drop for WriteLock {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
}
|
||||
}
|
||||
|
||||
struct BusyGate {
|
||||
entered: Option<oneshot::Sender<()>>,
|
||||
resume: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's
|
||||
/// userdata API can, without process globals or changing production hooks.
|
||||
struct BusyObservation {
|
||||
database: crate::FeatureDatabase,
|
||||
gate: Box<BusyGate>,
|
||||
resume: Option<mpsc::Sender<()>>,
|
||||
}
|
||||
impl BusyObservation {
|
||||
fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) {
|
||||
unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 {
|
||||
// SAFETY: owned Box remains registered until Drop unregisters it,
|
||||
// serialized by the managed connection mutex.
|
||||
let gate = unsafe { &mut *context.cast::<BusyGate>() };
|
||||
if let Some(entered) = gate.entered.take() {
|
||||
let _ = entered.send(());
|
||||
return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok());
|
||||
}
|
||||
0
|
||||
}
|
||||
let (entered, waiting) = oneshot::channel();
|
||||
let (resume, permit) = mpsc::channel();
|
||||
let mut gate = Box::new(BusyGate {
|
||||
entered: Some(entered),
|
||||
resume: permit,
|
||||
});
|
||||
database
|
||||
.with_connection(|conn| {
|
||||
// SAFETY: handle belongs to this connection; the Box is stable.
|
||||
let code = unsafe {
|
||||
ffi::sqlite3_busy_handler(
|
||||
conn.handle(),
|
||||
Some(busy),
|
||||
(&mut *gate as *mut BusyGate).cast(),
|
||||
)
|
||||
};
|
||||
assert_eq!(code, ffi::SQLITE_OK);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
(
|
||||
Self {
|
||||
database,
|
||||
gate,
|
||||
resume: Some(resume),
|
||||
},
|
||||
waiting,
|
||||
)
|
||||
}
|
||||
fn resume(&mut self) {
|
||||
if let Some(resume) = self.resume.take() {
|
||||
let _ = resume.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Drop for BusyObservation {
|
||||
fn drop(&mut self) {
|
||||
self.resume();
|
||||
self.database
|
||||
.with_connection(|conn| {
|
||||
// SAFETY: unregister before the boxed userdata is freed.
|
||||
unsafe {
|
||||
ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut());
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let _ = &self.gate;
|
||||
}
|
||||
}
|
||||
|
||||
// A handler with one DB stage can expose its admission without adding a test
|
||||
// hook to production. Polling its future to Pending proves run accepted it.
|
||||
struct AdmissionObservation(Mutex<Option<oneshot::Sender<()>>>);
|
||||
impl DbOperationLifetime for AdmissionObservation {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
if let Some(accepted) = self.0.lock().unwrap().take() {
|
||||
let _ = accepted.send(());
|
||||
}
|
||||
Box::new(())
|
||||
}
|
||||
}
|
||||
fn admission_observation() -> (Arc<dyn DbOperationLifetime>, oneshot::Receiver<()>) {
|
||||
let (accepted, receiver) = oneshot::channel();
|
||||
(
|
||||
Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))),
|
||||
receiver,
|
||||
)
|
||||
}
|
||||
|
||||
struct StateCheckSaturation {
|
||||
executor: DbExecutor,
|
||||
stages: std::sync::atomic::AtomicUsize,
|
||||
releases: Arc<Mutex<Vec<mpsc::Sender<()>>>>,
|
||||
}
|
||||
impl DbOperationLifetime for StateCheckSaturation {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
if self.stages.fetch_add(1, Ordering::SeqCst) == 1 {
|
||||
// Authentication has completed; saturate just before the lifecycle
|
||||
// lookup is admitted. The callback's own dispatch admissions are
|
||||
// distinct stages and do not recurse into this branch.
|
||||
let (started, running) = mpsc::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
self.executor
|
||||
.dispatch(move || {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv_timeout(WATCHDOG);
|
||||
})
|
||||
.unwrap();
|
||||
running
|
||||
.recv_timeout(WATCHDOG)
|
||||
.expect("state-check blocker did not start");
|
||||
self.releases.lock().unwrap().push(release);
|
||||
let (release, permit) = mpsc::channel();
|
||||
self.executor
|
||||
.dispatch(move || {
|
||||
let _ = permit.recv_timeout(WATCHDOG);
|
||||
})
|
||||
.unwrap();
|
||||
self.releases.lock().unwrap().push(release);
|
||||
}
|
||||
Box::new(())
|
||||
}
|
||||
}
|
||||
struct ReleaseStateCheckBlockers(Arc<Mutex<Vec<mpsc::Sender<()>>>>);
|
||||
impl Drop for ReleaseStateCheckBlockers {
|
||||
fn drop(&mut self) {
|
||||
self.0.lock().unwrap().clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() {
|
||||
let fixture = Fixture::new().await;
|
||||
let releases = Arc::new(Mutex::new(Vec::new()));
|
||||
let release = ReleaseStateCheckBlockers(releases.clone());
|
||||
let source = Arc::new(StateCheckSaturation {
|
||||
executor: fixture.executor.clone(),
|
||||
stages: std::sync::atomic::AtomicUsize::new(0),
|
||||
releases,
|
||||
});
|
||||
let router = Router::new()
|
||||
.route(
|
||||
SUBJECTS,
|
||||
axum::routing::post(|| async { StatusCode::NO_CONTENT }),
|
||||
)
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
fixture.api.clone(),
|
||||
authorize_workspace_api_request,
|
||||
));
|
||||
let response = observed(crate::db_executor::with_operation_lifetime(
|
||||
source,
|
||||
router.oneshot(Fixture::subject_request("not-executed")),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(release);
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await;
|
||||
assert!(
|
||||
!body.to_string().contains("Workspace is deleting"),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn runtime_progress_while_locked(lock: &WriteLock) {
|
||||
let control = tokio::spawn(async { "control progressed" });
|
||||
assert_eq!(observed(control).await.unwrap(), "control progressed");
|
||||
// The timer is the contract, not a guessed wait for request completion.
|
||||
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
|
||||
assert!(
|
||||
!lock.released.load(Ordering::SeqCst),
|
||||
"SQLite watchdog released the lock before runtime progress"
|
||||
);
|
||||
}
|
||||
async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value {
|
||||
let status = response.status();
|
||||
let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes));
|
||||
serde_json::from_slice(&bytes).expect("typed JSON response")
|
||||
}
|
||||
fn subject_count(path: &Path, role: &str) -> i64 {
|
||||
Connection::open(path)
|
||||
.unwrap()
|
||||
.query_row(
|
||||
"SELECT count(*) FROM subjects WHERE role = ?1",
|
||||
[role],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() {
|
||||
let fixture = Fixture::new().await;
|
||||
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
|
||||
let (lifetime, accepted) = admission_observation();
|
||||
let request = Fixture::request(
|
||||
"POST",
|
||||
"/api/w/db-execution/tickets",
|
||||
serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(),
|
||||
);
|
||||
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
lifetime,
|
||||
fixture.inner_router().oneshot(request),
|
||||
));
|
||||
assert!(futures::poll!(&mut response).is_pending());
|
||||
observed(accepted).await.unwrap();
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert!(
|
||||
futures::poll!(&mut response).is_pending(),
|
||||
"mutation responded before the SQLite lock was released"
|
||||
);
|
||||
let reader = Connection::open(&fixture.api.config.database_path).unwrap();
|
||||
let count: i64 = reader
|
||||
.query_row(
|
||||
"SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1",
|
||||
[WORKSPACE],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(count, 0, "Ticket must not be visible before commit");
|
||||
lock.release();
|
||||
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
|
||||
assert!(body["id"].is_string(), "{body}");
|
||||
// Ticket dispatch can call Runtime callbacks, so its operation owner is
|
||||
// the separate bounded host executor, not the store executor.
|
||||
observed(host_io_executor().unwrap().drain()).await.unwrap();
|
||||
fixture.executor.drain().await.unwrap();
|
||||
let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap();
|
||||
assert_eq!(count, 1, "one accepted HTTP mutation must commit once");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
drop(database); // FeatureStorage retains Weak handles: the route must cold-open.
|
||||
let mut lock = WriteLock::acquire(path);
|
||||
let (lifetime, accepted) = admission_observation();
|
||||
let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null);
|
||||
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
lifetime,
|
||||
fixture.inner_router().oneshot(request),
|
||||
));
|
||||
assert!(futures::poll!(&mut response).is_pending());
|
||||
observed(accepted).await.unwrap();
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert!(
|
||||
futures::poll!(&mut response).is_pending(),
|
||||
"cold open must wait for its SQLite scope transaction"
|
||||
);
|
||||
lock.release();
|
||||
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
|
||||
assert_eq!(body["items"], serde_json::json!([]), "{body}");
|
||||
fixture.executor.drain().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
let mut lock = WriteLock::acquire(path.clone());
|
||||
let (mut busy, waiting) = BusyObservation::install(database);
|
||||
let response = fixture.dispatch(Fixture::subject_request("locked-subject"));
|
||||
observed(waiting).await.unwrap();
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert!(
|
||||
!response.is_finished(),
|
||||
"HTTP response must not publish an uncommitted subject"
|
||||
);
|
||||
assert_eq!(subject_count(&path, "locked-subject"), 0);
|
||||
lock.release();
|
||||
busy.resume();
|
||||
let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await;
|
||||
assert_eq!(body["role"], "locked-subject", "{body}");
|
||||
assert!(
|
||||
body["memory_fingerprint"].is_string(),
|
||||
"response assembly must finish in the DB operation: {body}"
|
||||
);
|
||||
fixture.executor.drain().await.unwrap();
|
||||
assert_eq!(subject_count(&path, "locked-subject"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
let mut lock = WriteLock::acquire(path.clone());
|
||||
let (mut busy, waiting) = BusyObservation::install(database);
|
||||
let response = fixture.dispatch(Fixture::subject_request("cancelled-subject"));
|
||||
observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling.
|
||||
response.abort();
|
||||
assert!(observed(response).await.unwrap_err().is_cancelled());
|
||||
let admission = fixture.server.admission_for_workspace(WORKSPACE).await;
|
||||
let mut deletion = Box::pin(admission.lock_deletion());
|
||||
assert!(
|
||||
futures::poll!(&mut deletion).is_pending(),
|
||||
"aborting HTTP must not drop the accepted SQL RequestLease"
|
||||
);
|
||||
assert!(
|
||||
admission.admit(false).is_none(),
|
||||
"deletion must fence fresh requests"
|
||||
);
|
||||
assert_eq!(subject_count(&path, "cancelled-subject"), 0);
|
||||
lock.release();
|
||||
busy.resume();
|
||||
fixture.executor.drain().await.unwrap();
|
||||
let deletion = observed(deletion).await;
|
||||
assert_eq!(
|
||||
subject_count(&path, "cancelled-subject"),
|
||||
1,
|
||||
"cancelled waiter must neither cancel nor retry accepted SQL"
|
||||
);
|
||||
drop(deletion);
|
||||
assert!(
|
||||
admission.admit(false).is_some(),
|
||||
"deletion lease cleanup must reopen admission"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
let mut lock = WriteLock::acquire(path.clone());
|
||||
let (mut busy, waiting) = BusyObservation::install(database);
|
||||
let running = fixture.dispatch(Fixture::subject_request("running-subject"));
|
||||
observed(waiting).await.unwrap();
|
||||
let (lifetime, accepted) = admission_observation();
|
||||
let mut queued = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
lifetime,
|
||||
fixture
|
||||
.inner_router()
|
||||
.oneshot(Fixture::subject_request("queued-subject")),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
observed(accepted).await.unwrap();
|
||||
drop(queued); // Accepted job still owns the only waiting slot.
|
||||
let rejected = observed(
|
||||
fixture
|
||||
.inner_router()
|
||||
.oneshot(Fixture::subject_request("rejected-subject")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await;
|
||||
let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap();
|
||||
assert!(
|
||||
error.message.contains("capacity"),
|
||||
"typed overload error: {body}"
|
||||
);
|
||||
assert_eq!(subject_count(&path, "queued-subject"), 0);
|
||||
assert_eq!(subject_count(&path, "rejected-subject"), 0);
|
||||
let mut drained = Box::pin(fixture.executor.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
lock.release();
|
||||
busy.resume();
|
||||
response_json(observed(running).await.unwrap(), StatusCode::CREATED).await;
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(
|
||||
subject_count(&path, "queued-subject"),
|
||||
1,
|
||||
"cancelled accepted mutation persists exactly once"
|
||||
);
|
||||
assert_eq!(
|
||||
subject_count(&path, "rejected-subject"),
|
||||
0,
|
||||
"overload must not execute rejected SQL"
|
||||
);
|
||||
let retry = fixture.dispatch(Fixture::subject_request("after-drain"));
|
||||
response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await;
|
||||
fixture.executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
subject_count(&path, "after-drain"),
|
||||
1,
|
||||
"drain must release actual-job capacity"
|
||||
);
|
||||
}
|
||||
@@ -51,14 +51,13 @@ async fn blocking<T: Send + 'static>(
|
||||
mutation: bool,
|
||||
action: impl FnOnce() -> workspace_drive::Result<T> + Send + 'static,
|
||||
) -> DriveResult<T> {
|
||||
tokio::task::spawn_blocking(action)
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || Ok::<_, Error>(action()))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
if mutation {
|
||||
DriveApiError::new(DriveApiErrorCode::OutcomeUnknown)
|
||||
} else {
|
||||
failure(503)
|
||||
}
|
||||
.map_err(|error| match error {
|
||||
Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503),
|
||||
_ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown),
|
||||
_ => failure(503),
|
||||
})?
|
||||
.map_err(|error| domain_error(error, mutation))
|
||||
}
|
||||
|
||||
@@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> DriveApiError {
|
||||
})
|
||||
}
|
||||
fn mutation_error(error: Error) -> DriveApiError {
|
||||
if matches!(error, Error::Sqlite(_) | Error::Store(_)) {
|
||||
if matches!(
|
||||
error,
|
||||
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
|
||||
) {
|
||||
fail(DriveApiErrorCode::OutcomeUnknown)
|
||||
} else {
|
||||
pre_error(error)
|
||||
@@ -97,9 +100,13 @@ pub(super) async fn authorized_drive(
|
||||
};
|
||||
// Reject missing/read-only grants before upload body reception. This check
|
||||
// is admission only: every later operation rechecks under its own lock.
|
||||
tokio::task::spawn_blocking(move || {
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
drive.check_authorized(write)?;
|
||||
Ok::<_, workspace_drive::Error>(drive)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
@@ -152,9 +159,10 @@ pub(super) async fn create_grant(
|
||||
revoked_at: None,
|
||||
};
|
||||
let store = api.store.clone();
|
||||
tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant))
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || store.create_workspace_drive_grant(&grant))
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
|
||||
.map_err(mutation_error)
|
||||
}
|
||||
|
||||
@@ -175,11 +183,10 @@ pub(super) async fn revoke_grant(
|
||||
actor.account_id.clone(),
|
||||
);
|
||||
let store = api.store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
store.revoke_workspace_drive_grant(&workspace, &grant, &actor)
|
||||
})
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor))
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
|
||||
.map_err(mutation_error)?
|
||||
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
|
||||
}
|
||||
@@ -212,11 +219,13 @@ pub(super) async fn list_grants(
|
||||
}
|
||||
let workspace = workspace_id.to_owned();
|
||||
let store = api.store.clone();
|
||||
let mut grants = tokio::task::spawn_blocking(move || {
|
||||
let mut grants = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
.map_err(pre_error)?;
|
||||
let more = grants.len() > limit;
|
||||
grants.truncate(limit);
|
||||
|
||||
@@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> Failure {
|
||||
fn mutation_error(error: Error) -> Failure {
|
||||
// Storage/service failures at a mutation boundary may include a commit
|
||||
// acknowledgement failure. Never describe those as a guaranteed rollback.
|
||||
if matches!(&error, Error::Sqlite(_) | Error::Store(_)) {
|
||||
if matches!(
|
||||
&error,
|
||||
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
|
||||
) {
|
||||
unknown()
|
||||
} else {
|
||||
pre_error(error)
|
||||
@@ -186,9 +189,23 @@ pub(super) async fn get(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ConfigResult<Option<WorkspaceConfigAttachment>> {
|
||||
let _guard = lock(api, worker).await?;
|
||||
let grant = grant_for_worker(api, worker)?;
|
||||
Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true)))
|
||||
let guard = lock(api, worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let worker = worker.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
let grant = grant_for_worker(&api, &worker)?;
|
||||
Ok::<_, Failure>(
|
||||
active_link(&api, &worker, &grant, None)?
|
||||
.map(|link| attachment(&grant, &link, true)),
|
||||
)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
}
|
||||
pub(super) async fn attach(
|
||||
api: &WorkspaceApi,
|
||||
@@ -198,20 +215,29 @@ pub(super) async fn attach(
|
||||
if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) {
|
||||
return Err(fail(400, "invalid_alias"));
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let grant = grant_for_worker(api, worker)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let worker = worker.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
let grant = grant_for_worker(&api, &worker)?;
|
||||
let access = request.access.unwrap_or(grant.access);
|
||||
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if let Some(mut link) = active_link(api, worker, &grant, None)? {
|
||||
if let Some(mut link) = active_link(&api, &worker, &grant, None)? {
|
||||
// Same active alias/identity is idempotent; explicit write never silently downgrades.
|
||||
if request.access == Some(Access::ReadWrite)
|
||||
&& effective_access(&grant, &link) != Access::ReadWrite
|
||||
{
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly {
|
||||
if access == Access::ReadOnly
|
||||
&& effective_access(&grant, &link) != Access::ReadOnly
|
||||
{
|
||||
link.capabilities = access_capabilities(access);
|
||||
link = api
|
||||
.store
|
||||
@@ -235,9 +261,13 @@ pub(super) async fn attach(
|
||||
.map_err(mutation_error)?;
|
||||
// No Runtime filesystem/session synchronization for a logical config target.
|
||||
api.worker_projection
|
||||
.refresh(worker)
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(attachment(&grant, &link, false))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
fn resolve(
|
||||
api: &WorkspaceApi,
|
||||
@@ -313,9 +343,22 @@ pub(super) async fn observe(
|
||||
for path in &request.paths {
|
||||
path_valid(path, true)?;
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, _guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -427,9 +470,22 @@ pub(super) async fn read(
|
||||
request: WorkspaceConfigReadRequest,
|
||||
) -> ConfigResult<WorkspaceConfigReadResponse> {
|
||||
path_valid(&request.path, false)?;
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, _guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -482,12 +538,25 @@ pub(super) async fn commit(
|
||||
server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?,
|
||||
}
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
if effective_access(&grant, &link) != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
let state = tree(api)?;
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -501,17 +570,19 @@ pub(super) async fn commit(
|
||||
// Evaluation/semantic validation are pure preparation. Bound this stage
|
||||
// without ever dispatching persistence in a timed-out background task.
|
||||
let preparation_api = api.clone();
|
||||
let preparation = tokio::task::spawn_blocking(move || {
|
||||
prepare_workspace_config_tree(
|
||||
let executor = api.store.db_executor();
|
||||
let preparation = executor.run(move || {
|
||||
// Timeout/cancellation releases exclusion only after actual preparation ends.
|
||||
let candidate = prepare_workspace_config_tree(
|
||||
&preparation_api,
|
||||
&preparation_api.config.workspace_id,
|
||||
&canonical,
|
||||
)
|
||||
)?;
|
||||
Ok::<_, Error>((candidate, guard))
|
||||
});
|
||||
let candidate = tokio::time::timeout(DEADLINE, preparation)
|
||||
let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation)
|
||||
.await
|
||||
.map_err(|_| fail(503, "evaluation_timeout"))?
|
||||
.map_err(|_| fail(500, "evaluation_unavailable"))?
|
||||
.map_err(pre_error)?;
|
||||
if candidate.base_digest != state.snapshot.digest
|
||||
|| candidate.base_toolchain_fingerprint != state.contract.fingerprint
|
||||
@@ -520,15 +591,78 @@ pub(super) async fn commit(
|
||||
{
|
||||
return Err(fail(409, "stale_validator"));
|
||||
}
|
||||
// Recheck durable authority before the first auxiliary/canonical effects.
|
||||
let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?;
|
||||
if current_grant != grant || current_link.capabilities != link.capabilities {
|
||||
let persistence_api = api.clone();
|
||||
let worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let expected_grant = grant.clone();
|
||||
let expected_link = link.clone();
|
||||
let (state, publication) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (current_grant, current_link) =
|
||||
resolve(&persistence_api, &worker, &connection_id)?;
|
||||
if current_grant != expected_grant
|
||||
|| current_link.capabilities != expected_link.capabilities
|
||||
{
|
||||
return Err(fail(409, "authority_changed"));
|
||||
}
|
||||
// From this boundary repository credential projection may have side effects.
|
||||
// Never falsely claim rollback or automatically replay a failed/uncertain save.
|
||||
let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate)
|
||||
// Preserve the shared helper's ordering and atomic CAS, but never
|
||||
// run its remote Runtime publication on the sole DB worker.
|
||||
project_repository_access_candidate(
|
||||
&*persistence_api.store,
|
||||
&persistence_api.repository_secrets,
|
||||
&persistence_api.config.workspace_id,
|
||||
&candidate,
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
let state = persistence_api
|
||||
.config_store
|
||||
.commit_evaluated_workspace_config(
|
||||
&persistence_api.config.workspace_id,
|
||||
&candidate,
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
let publication = persistence_api
|
||||
.prompt_projection_cache
|
||||
.resolve(&persistence_api.config.workspace_id, &state)
|
||||
.ok()
|
||||
.and_then(|projection| {
|
||||
// Dispatch from the actual persistence owner even if its
|
||||
// waiter was cancelled. Runtime I/O owns exclusion, not
|
||||
// a DB worker or an open SQL transaction.
|
||||
let (complete, publication) = tokio::sync::oneshot::channel();
|
||||
let runtime = persistence_api.runtime.clone();
|
||||
let admission = host_io_executor().and_then(|executor| {
|
||||
executor.dispatch(move || {
|
||||
let _guard = guard;
|
||||
let _diagnostics = runtime
|
||||
.observe_workspace_prompt_projection((*projection).clone());
|
||||
let _ = complete.send(());
|
||||
})
|
||||
});
|
||||
match admission {
|
||||
Ok(()) => Some(publication),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
workspace_id = %persistence_api.config.workspace_id,
|
||||
%error,
|
||||
"Workspace config committed but Runtime publication was not admitted"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok((state, publication))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)??;
|
||||
if let Some(publication) = publication {
|
||||
// Publication is best effort and cannot turn an acknowledged CAS into an error.
|
||||
let _ = publication.await;
|
||||
}
|
||||
Ok(WorkspaceConfigCommitResponse {
|
||||
validator: validator(&grant, &link, &state, &schema_fingerprint, ""),
|
||||
digest: state.snapshot.digest,
|
||||
@@ -544,17 +678,35 @@ pub(super) async fn create_grant(
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id);
|
||||
let _guard = lock(api, &worker).await?;
|
||||
let summary = api
|
||||
.runtime
|
||||
.worker(&worker)
|
||||
let guard = lock(api, &worker).await?;
|
||||
let runtime = api.runtime.clone();
|
||||
let lookup_worker = worker.clone();
|
||||
let (summary, guard) = host_io_executor()
|
||||
.map_err(pre_error)?
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let summary = runtime
|
||||
.worker(&lookup_worker)
|
||||
.map_err(|_| fail(404, "worker_not_found"))?;
|
||||
Ok::<_, Failure>((summary, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
if summary.workspace.workspace_id.as_deref() != Some(workspace_id) {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let account_id = actor.account_id.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
if let Some(grant) = api
|
||||
.store
|
||||
.current_workspace_config_grant(workspace_id, &worker)
|
||||
.current_workspace_config_grant(&workspace_id, &worker)
|
||||
.map_err(pre_error)?
|
||||
{
|
||||
if grant.access != request.access {
|
||||
@@ -564,7 +716,7 @@ pub(super) async fn create_grant(
|
||||
}
|
||||
let grant = WorkspaceConfigGrantResponse {
|
||||
grant_id: Uuid::now_v7().to_string(),
|
||||
workspace_id: workspace_id.into(),
|
||||
workspace_id,
|
||||
runtime_id: worker.runtime_id,
|
||||
worker_id: worker.worker_id,
|
||||
working_directory_id: Uuid::now_v7().to_string(),
|
||||
@@ -572,9 +724,13 @@ pub(super) async fn create_grant(
|
||||
revoked: false,
|
||||
};
|
||||
api.store
|
||||
.create_workspace_config_grant(&grant, &actor.account_id)
|
||||
.create_workspace_config_grant(&grant, &account_id)
|
||||
.map_err(mutation_error)?;
|
||||
Ok(grant)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
pub(super) async fn revoke_grant(
|
||||
api: &WorkspaceApi,
|
||||
@@ -585,28 +741,45 @@ pub(super) async fn revoke_grant(
|
||||
require_workspace_owner(api, workspace_id, actor, "Workspace config grants")
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let lookup_api = api.clone();
|
||||
let lookup_workspace_id = workspace_id.to_owned();
|
||||
let lookup_grant_id = grant_id.to_owned();
|
||||
let mut grant = api
|
||||
.store
|
||||
.get_workspace_config_grant(workspace_id, grant_id)
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
lookup_api
|
||||
.store
|
||||
.get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id)
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
.ok_or_else(|| fail(404, "not_found"))?;
|
||||
let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id);
|
||||
let _guard = lock(api, &worker).await?;
|
||||
let guard = lock(api, &worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let grant_id = grant_id.to_owned();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
api.store
|
||||
.revoke_workspace_config_grant(workspace_id, grant_id)
|
||||
.revoke_workspace_config_grant(&workspace_id, &grant_id)
|
||||
.map_err(mutation_error)?;
|
||||
grant.revoked = true;
|
||||
// Revocation is durable before unlink; a failed cleanup cannot restore access.
|
||||
if let Some(link) = api
|
||||
.store
|
||||
.list_workdir_worker_links(workspace_id, &grant.working_directory_id)
|
||||
.list_workdir_worker_links(&workspace_id, &grant.working_directory_id)
|
||||
.map_err(|_| unknown())?
|
||||
.into_iter()
|
||||
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
|
||||
{
|
||||
api.store
|
||||
.detach_worker_workdir_connection(
|
||||
workspace_id,
|
||||
&workspace_id,
|
||||
&worker,
|
||||
&link.alias,
|
||||
&link.connection_id,
|
||||
@@ -618,6 +791,10 @@ pub(super) async fn revoke_grant(
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(grant)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
|
||||
pub(super) fn summary(
|
||||
@@ -655,7 +832,11 @@ pub(super) fn summary(
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn workspace_config_mutation_storage_failures_are_conservatively_unknown() {
|
||||
fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() {
|
||||
assert_eq!(
|
||||
mutation_error(Error::DbOperationOutcomeUnknown).classification,
|
||||
Classification::Unknown
|
||||
);
|
||||
assert_eq!(
|
||||
mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification,
|
||||
Classification::Unknown
|
||||
|
||||
@@ -803,9 +803,9 @@ mod workspace_config_integration {
|
||||
.supports(workdir::WorkdirSessionCapability::Command)
|
||||
);
|
||||
let lock = current_worker_session_lock(api, &fixture.worker);
|
||||
let _guard = lock.lock().await;
|
||||
let _guard = Arc::new(lock.lock_owned().await);
|
||||
assert!(
|
||||
open_current_worker_workdir_session_locked(api, &fixture.worker, &link)
|
||||
open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
@@ -1123,6 +1123,10 @@ impl WorkspaceResourceKind {
|
||||
|
||||
#[async_trait]
|
||||
pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
||||
fn db_executor(&self) -> crate::db_executor::DbExecutor {
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
}
|
||||
|
||||
async fn schema_version(&self) -> Result<i64>;
|
||||
fn resource_key(
|
||||
&self,
|
||||
@@ -2179,6 +2183,7 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
||||
#[derive(Clone)]
|
||||
pub struct SqliteWorkspaceStore {
|
||||
conn: Arc<Mutex<Connection>>,
|
||||
db_executor: crate::db_executor::DbExecutor,
|
||||
}
|
||||
|
||||
impl SqliteWorkspaceStore {
|
||||
@@ -2222,9 +2227,20 @@ impl SqliteWorkspaceStore {
|
||||
prepare_connection(&conn)?;
|
||||
Ok(Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Override execution limits for a hermetic fixture; subsequent clones share it.
|
||||
pub fn with_db_executor(mut self, executor: crate::db_executor::DbExecutor) -> Self {
|
||||
self.db_executor = executor;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn db_executor(&self) -> crate::db_executor::DbExecutor {
|
||||
self.db_executor.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn with_conn<T>(&self, f: impl FnOnce(&Connection) -> Result<T>) -> Result<T> {
|
||||
let conn = self
|
||||
.conn
|
||||
@@ -3794,8 +3810,16 @@ fn insert_workspace_runtime_binding_audit(
|
||||
|
||||
#[async_trait]
|
||||
impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
fn db_executor(&self) -> crate::db_executor::DbExecutor {
|
||||
self.db_executor.clone()
|
||||
}
|
||||
|
||||
async fn schema_version(&self) -> Result<i64> {
|
||||
self.with_conn(current_schema_version)
|
||||
let store = self.clone();
|
||||
|
||||
self.db_executor
|
||||
.run(move || store.with_conn(current_schema_version))
|
||||
.await
|
||||
}
|
||||
|
||||
fn resource_key(
|
||||
@@ -3927,7 +3951,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
}
|
||||
|
||||
async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()> {
|
||||
self.with_conn_mut(|conn| {
|
||||
let store = self.clone();
|
||||
let record = record.clone();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let record = &record;
|
||||
store.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
#[cfg(test)]
|
||||
if record.owner_account_id == "owner-account" {
|
||||
@@ -4007,11 +4037,18 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})?;
|
||||
self.materialize_workspace_config(&record.workspace_id, &record.created_at)
|
||||
store.materialize_workspace_config(&record.workspace_id, &record.created_at)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_workspace(&self, workspace_id: &str) -> Result<Option<WorkspaceRecord>> {
|
||||
self.with_conn(|conn| {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
self.db_executor.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
store.with_conn(|conn| {
|
||||
conn.query_row(
|
||||
r#"SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at
|
||||
FROM workspaces WHERE workspace_id = ?1"#,
|
||||
@@ -4021,6 +4058,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
.optional()
|
||||
.map_err(Error::from)
|
||||
})
|
||||
}).await
|
||||
}
|
||||
|
||||
async fn update_workspace_display_name(
|
||||
@@ -4029,10 +4067,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
expected_updated_at: &str,
|
||||
display_name: &str,
|
||||
) -> Result<Option<WorkspaceRecord>> {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let expected_updated_at = expected_updated_at.to_owned();
|
||||
let display_name = display_name.to_owned();
|
||||
self.db_executor.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let expected_updated_at = expected_updated_at.as_str();
|
||||
let display_name = display_name.as_str();
|
||||
validate_identifier("workspace_id", workspace_id)?;
|
||||
validate_non_empty("expected_updated_at", expected_updated_at)?;
|
||||
validate_non_empty("display_name", display_name)?;
|
||||
self.with_conn_mut(|conn| {
|
||||
store.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let current = tx
|
||||
.query_row(
|
||||
@@ -4081,6 +4128,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
tx.commit()?;
|
||||
Ok(Some(updated))
|
||||
})
|
||||
}).await
|
||||
}
|
||||
|
||||
fn create_workspace_bootstrap(
|
||||
@@ -4600,7 +4648,17 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
workspace_id: &str,
|
||||
runtime_id: &str,
|
||||
) -> Result<Option<WorkspaceRuntimeBinding>> {
|
||||
SqliteWorkspaceStore::get_workspace_runtime_binding(self, workspace_id, runtime_id)
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
SqliteWorkspaceStore::get_workspace_runtime_binding(store, workspace_id, runtime_id)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_workspace_runtime_bindings(
|
||||
@@ -4608,7 +4666,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
workspace_id: &str,
|
||||
include_revoked: bool,
|
||||
) -> Result<Vec<WorkspaceRuntimeBinding>> {
|
||||
SqliteWorkspaceStore::list_workspace_runtime_bindings(self, workspace_id, include_revoked)
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
SqliteWorkspaceStore::list_workspace_runtime_bindings(
|
||||
store,
|
||||
workspace_id,
|
||||
include_revoked,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn has_other_active_workspace_runtime_binding(
|
||||
@@ -4616,11 +4686,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
workspace_id: &str,
|
||||
runtime_id: &str,
|
||||
) -> Result<bool> {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
SqliteWorkspaceStore::has_other_active_workspace_runtime_binding(
|
||||
self,
|
||||
store,
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_workspace_runtime_binding(
|
||||
@@ -4628,7 +4708,21 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
workspace_id: &str,
|
||||
runtime_id: &str,
|
||||
) -> Result<bool> {
|
||||
SqliteWorkspaceStore::delete_workspace_runtime_binding(self, workspace_id, runtime_id)
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
SqliteWorkspaceStore::delete_workspace_runtime_binding(
|
||||
store,
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_runtime_removal(
|
||||
@@ -4640,8 +4734,24 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
expected_binding_id: &str,
|
||||
config_digest: &str,
|
||||
) -> Result<RuntimeRemovalReservation> {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let operation_id = operation_id.to_owned();
|
||||
let request_fingerprint = request_fingerprint.to_owned();
|
||||
let expected_binding_id = expected_binding_id.to_owned();
|
||||
let config_digest = config_digest.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
let operation_id = operation_id.as_str();
|
||||
let request_fingerprint = request_fingerprint.as_str();
|
||||
let expected_binding_id = expected_binding_id.as_str();
|
||||
let config_digest = config_digest.as_str();
|
||||
SqliteWorkspaceStore::reserve_runtime_removal(
|
||||
self,
|
||||
store,
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
operation_id,
|
||||
@@ -4649,13 +4759,23 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
expected_binding_id,
|
||||
config_digest,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_runtime_removal(
|
||||
&self,
|
||||
operation_id: &str,
|
||||
) -> Result<Option<RuntimeRemovalOperation>> {
|
||||
SqliteWorkspaceStore::get_runtime_removal(self, operation_id)
|
||||
let store = self.clone();
|
||||
let operation_id = operation_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let operation_id = operation_id.as_str();
|
||||
SqliteWorkspaceStore::get_runtime_removal(store, operation_id)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn mark_runtime_removal_failed(
|
||||
@@ -4663,14 +4783,36 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
operation_id: &str,
|
||||
failure_category: &str,
|
||||
) -> Result<RuntimeRemovalOperation> {
|
||||
SqliteWorkspaceStore::mark_runtime_removal_failed(self, operation_id, failure_category)
|
||||
let store = self.clone();
|
||||
let operation_id = operation_id.to_owned();
|
||||
let failure_category = failure_category.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let operation_id = operation_id.as_str();
|
||||
let failure_category = failure_category.as_str();
|
||||
SqliteWorkspaceStore::mark_runtime_removal_failed(
|
||||
store,
|
||||
operation_id,
|
||||
failure_category,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn commit_runtime_binding_removal(
|
||||
&self,
|
||||
operation_id: &str,
|
||||
) -> Result<RuntimeRemovalOperation> {
|
||||
SqliteWorkspaceStore::commit_runtime_binding_removal(self, operation_id)
|
||||
let store = self.clone();
|
||||
let operation_id = operation_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let operation_id = operation_id.as_str();
|
||||
SqliteWorkspaceStore::commit_runtime_binding_removal(store, operation_id)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn complete_runtime_removal(
|
||||
@@ -4678,18 +4820,34 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
operation_id: &str,
|
||||
runtime_registration_removed: bool,
|
||||
) -> Result<RuntimeRemovalOperation> {
|
||||
let store = self.clone();
|
||||
let operation_id = operation_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let operation_id = operation_id.as_str();
|
||||
SqliteWorkspaceStore::complete_runtime_removal(
|
||||
self,
|
||||
store,
|
||||
operation_id,
|
||||
runtime_registration_removed,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_resumable_runtime_removals(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
) -> Result<Vec<RuntimeRemovalOperation>> {
|
||||
SqliteWorkspaceStore::list_resumable_runtime_removals(self, workspace_id)
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
SqliteWorkspaceStore::list_resumable_runtime_removals(store, workspace_id)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn upsert_workspace_runtime_binding_record(
|
||||
@@ -4697,7 +4855,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
record: WorkspaceRuntimeBinding,
|
||||
replace: bool,
|
||||
) -> Result<WorkspaceRuntimeBindingUpsert> {
|
||||
SqliteWorkspaceStore::upsert_workspace_runtime_binding(self, record, replace)
|
||||
let store = self.clone();
|
||||
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
SqliteWorkspaceStore::upsert_workspace_runtime_binding(store, record, replace)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn revoke_workspace_runtime_binding_record(
|
||||
@@ -4706,12 +4871,24 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
runtime_id: &str,
|
||||
revoked_at: &str,
|
||||
) -> Result<bool> {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let revoked_at = revoked_at.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
let revoked_at = revoked_at.as_str();
|
||||
SqliteWorkspaceStore::revoke_workspace_runtime_binding(
|
||||
self,
|
||||
store,
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
revoked_at,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_workspace_runtime_binding_key(
|
||||
@@ -4720,12 +4897,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
expected_binding_id: Option<&str>,
|
||||
actor_account_id: &str,
|
||||
) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> {
|
||||
let store = self.clone();
|
||||
let expected_binding_id = expected_binding_id.map(str::to_owned);
|
||||
let actor_account_id = actor_account_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let expected_binding_id = expected_binding_id.as_deref();
|
||||
let actor_account_id = actor_account_id.as_str();
|
||||
SqliteWorkspaceStore::put_workspace_runtime_binding_key(
|
||||
self,
|
||||
store,
|
||||
record,
|
||||
expected_binding_id,
|
||||
actor_account_id,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_workspace_runtime_binding_metadata(
|
||||
@@ -4736,14 +4923,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
base_url: &str,
|
||||
updated_at: &str,
|
||||
) -> Result<WorkspaceRuntimeBinding> {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let display_name = display_name.to_owned();
|
||||
let base_url = base_url.to_owned();
|
||||
let updated_at = updated_at.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
let display_name = display_name.as_str();
|
||||
let base_url = base_url.as_str();
|
||||
let updated_at = updated_at.as_str();
|
||||
SqliteWorkspaceStore::update_workspace_runtime_binding_metadata(
|
||||
self,
|
||||
store,
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
display_name,
|
||||
base_url,
|
||||
updated_at,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn revoke_workspace_runtime_binding_key(
|
||||
@@ -4754,14 +4957,30 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
actor_account_id: &str,
|
||||
revoked_at: &str,
|
||||
) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let expected_binding_id = expected_binding_id.to_owned();
|
||||
let actor_account_id = actor_account_id.to_owned();
|
||||
let revoked_at = revoked_at.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
let expected_binding_id = expected_binding_id.as_str();
|
||||
let actor_account_id = actor_account_id.as_str();
|
||||
let revoked_at = revoked_at.as_str();
|
||||
SqliteWorkspaceStore::revoke_workspace_runtime_binding_key(
|
||||
self,
|
||||
store,
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
expected_binding_id,
|
||||
actor_account_id,
|
||||
revoked_at,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_workspace_runtime_binding_audit(
|
||||
@@ -4770,12 +4989,22 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
runtime_id: &str,
|
||||
limit: usize,
|
||||
) -> Result<Vec<WorkspaceRuntimeBindingAuditRecord>> {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
SqliteWorkspaceStore::list_workspace_runtime_binding_audit(
|
||||
self,
|
||||
store,
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
limit,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn consume_worker_mutation_source_jti(
|
||||
@@ -4787,7 +5016,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
now_seconds: u64,
|
||||
consumed_at: &str,
|
||||
) -> Result<bool> {
|
||||
self.with_conn(|conn| {
|
||||
let store = self.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let jti = jti.to_owned();
|
||||
let consumed_at = consumed_at.to_owned();
|
||||
self.db_executor
|
||||
.run(move || {
|
||||
let store = &store;
|
||||
let workspace_id = workspace_id.as_str();
|
||||
let runtime_id = runtime_id.as_str();
|
||||
let jti = jti.as_str();
|
||||
let consumed_at = consumed_at.as_str();
|
||||
store.with_conn(|conn| {
|
||||
let transaction = conn.unchecked_transaction()?;
|
||||
transaction.execute(
|
||||
"DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1",
|
||||
@@ -4802,6 +5043,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
transaction.commit()?;
|
||||
Ok(inserted == 1)
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn plan_worker_removal(
|
||||
|
||||
@@ -5,7 +5,7 @@ use chrono::Utc;
|
||||
use protocol::subscription::{
|
||||
EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot,
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
use tokio::sync::{Semaphore, broadcast};
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use crate::runtime_subscription::{
|
||||
@@ -59,7 +59,14 @@ pub struct WorkerProjectionService {
|
||||
events: broadcast::Sender<WorkerProjectionEvent>,
|
||||
ordering: Mutex<()>,
|
||||
attach_gate: tokio::sync::Mutex<()>,
|
||||
tasks: Mutex<HashMap<String, JoinHandle<()>>>,
|
||||
tasks: Mutex<HashMap<String, RuntimeProjectionLifetime>>,
|
||||
}
|
||||
|
||||
struct RuntimeProjectionLifetime {
|
||||
consumer: Option<JoinHandle<()>>,
|
||||
// The permit belongs to the actual DB closure, not its cancellable async waiter.
|
||||
// Keep this owner in the map even if attachment initialization is cancelled.
|
||||
operations: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl WorkerProjectionService {
|
||||
@@ -75,16 +82,88 @@ impl WorkerProjectionService {
|
||||
})
|
||||
}
|
||||
|
||||
/// Stops consumers but does not claim that accepted database work has finished.
|
||||
/// Async shutdown callers must use `shutdown_async` to drain accepted operations.
|
||||
pub fn shutdown(&self) {
|
||||
for (_, task) in self
|
||||
for lifetime in self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.drain()
|
||||
.values()
|
||||
{
|
||||
if let Some(task) = &lifetime.consumer {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn shutdown_async(&self) {
|
||||
let _attach = self.attach_gate.lock().await;
|
||||
self.shutdown();
|
||||
let runtime_ids = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for runtime_id in runtime_ids {
|
||||
self.drain_runtime(&runtime_id).await;
|
||||
}
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.clear();
|
||||
}
|
||||
|
||||
async fn drain_runtime(&self, runtime_id: &str) {
|
||||
let previous = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.get_mut(runtime_id)
|
||||
.map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone()));
|
||||
if let Some((consumer, operations)) = previous {
|
||||
if let Some(consumer) = consumer {
|
||||
consumer.abort();
|
||||
let _ = consumer.await;
|
||||
}
|
||||
// Aborting the waiter cannot abort a queued/running synchronous job.
|
||||
// Cancellation here leaves the gate in the map for the next drain.
|
||||
let _drained = operations
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("operation gate closed");
|
||||
}
|
||||
}
|
||||
|
||||
// Caller holds attach_gate. Install ownership before any initialization await.
|
||||
async fn replace_runtime(&self, runtime_id: &str) -> Arc<Semaphore> {
|
||||
self.drain_runtime(runtime_id).await;
|
||||
let operations = Arc::new(Semaphore::new(1));
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.insert(
|
||||
runtime_id.to_string(),
|
||||
RuntimeProjectionLifetime {
|
||||
consumer: None,
|
||||
operations: operations.clone(),
|
||||
},
|
||||
);
|
||||
operations
|
||||
}
|
||||
|
||||
pub async fn subscribe_ordered_async(
|
||||
self: &Arc<Self>,
|
||||
limit: usize,
|
||||
) -> crate::Result<WorkerProjectionSubscription> {
|
||||
let service = self.clone();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.subscribe_ordered(limit))
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn subscribe_ordered(&self, limit: usize) -> crate::Result<WorkerProjectionSubscription> {
|
||||
let _ordering = self
|
||||
@@ -125,19 +204,10 @@ impl WorkerProjectionService {
|
||||
broker: &RuntimeSubscriptionBroker,
|
||||
runtime_id: &str,
|
||||
) -> Result<(), RuntimeSubscriptionBrokerError> {
|
||||
// Serialize replacement and wait for the old consumer to stop before the new
|
||||
// lifetime can apply its snapshot. Late notifications can therefore never
|
||||
// cross a Runtime ownership boundary.
|
||||
// Serialize replacement, stop admission by aborting the consumer, and then
|
||||
// drain ownership held by the actual SQL jobs before the new snapshot.
|
||||
let _attach = self.attach_gate.lock().await;
|
||||
let previous = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.remove(runtime_id);
|
||||
if let Some(previous) = previous {
|
||||
previous.abort();
|
||||
let _ = previous.await;
|
||||
}
|
||||
let operations = self.replace_runtime(runtime_id).await;
|
||||
|
||||
let mut subscription =
|
||||
broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?;
|
||||
@@ -148,12 +218,25 @@ impl WorkerProjectionService {
|
||||
break;
|
||||
};
|
||||
let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. });
|
||||
if let Err(error) = self.apply_broker_event(runtime_id, event) {
|
||||
if let Err(error) = self
|
||||
.apply_broker_event_owned_async(runtime_id, event, operations.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
runtime_id,
|
||||
error = %error,
|
||||
"failed to initialize durable Worker projection; restarting from a fresh snapshot"
|
||||
);
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbCapacityExceeded
|
||||
) {
|
||||
// Do not automatically replay a possibly committed mutation or
|
||||
// spin on executor overload; attachment must be retried explicitly.
|
||||
return Err(RuntimeSubscriptionBrokerError::Closed);
|
||||
}
|
||||
subscription.restart()?;
|
||||
continue;
|
||||
}
|
||||
@@ -169,12 +252,27 @@ impl WorkerProjectionService {
|
||||
let Some(service) = service.upgrade() else {
|
||||
return;
|
||||
};
|
||||
if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) {
|
||||
if let Err(error) = service
|
||||
.apply_broker_event_owned_async(
|
||||
task_runtime_id.as_str(),
|
||||
event,
|
||||
operations.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
runtime_id = %task_runtime_id,
|
||||
error = %error,
|
||||
"failed to update durable Worker projection; restarting from a fresh snapshot"
|
||||
);
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbCapacityExceeded
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if subscription.restart().is_err() {
|
||||
return;
|
||||
}
|
||||
@@ -184,10 +282,61 @@ impl WorkerProjectionService {
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.insert(runtime_id, task);
|
||||
.get_mut(&runtime_id)
|
||||
.expect("Runtime lifetime installed before initialization")
|
||||
.consumer = Some(task);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn seed_observation_async(
|
||||
self: &Arc<Self>,
|
||||
runtime_id: &str,
|
||||
worker: &protocol::subscription::SubscriptionWorker,
|
||||
observed_at: &str,
|
||||
) -> crate::Result<()> {
|
||||
let service = self.clone();
|
||||
let runtime_id = runtime_id.to_string();
|
||||
let worker = worker.clone();
|
||||
let observed_at = observed_at.to_string();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.seed_observation(&runtime_id, &worker, &observed_at))
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn refresh_async(self: &Arc<Self>, worker: &RuntimeWorkerRef) -> crate::Result<()> {
|
||||
let service = self.clone();
|
||||
let worker = worker.clone();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.refresh(&worker))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn apply_broker_event_owned_async(
|
||||
self: &Arc<Self>,
|
||||
runtime_id: &str,
|
||||
event: BrokerSubscriptionEvent,
|
||||
operations: Arc<Semaphore>,
|
||||
) -> crate::Result<()> {
|
||||
let operation = operations
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("operation gate closed");
|
||||
let service = self.clone();
|
||||
let runtime_id = runtime_id.to_string();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
// The permit must outlive SQL commit AND ordered publication, even
|
||||
// when the executor result receiver has been dropped by cancellation.
|
||||
let _operation = operation;
|
||||
service.apply_broker_event(&runtime_id, event)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn seed_observation(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -313,7 +462,11 @@ mod tests {
|
||||
tempfile::TempDir,
|
||||
) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap());
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open(temp.path().join("workspace.db"))
|
||||
.unwrap()
|
||||
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".to_string(),
|
||||
@@ -355,6 +508,266 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn observation(
|
||||
state: protocol::subscription::SubscriptionWorkerState,
|
||||
) -> protocol::subscription::SubscriptionWorker {
|
||||
protocol::subscription::SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(),
|
||||
runtime_id: Some("runtime-a".to_string()),
|
||||
resource_key: None,
|
||||
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
|
||||
worker_state: None,
|
||||
state,
|
||||
has_running_internal_workers: false,
|
||||
workspace_id: Some(WORKSPACE_ID.to_string()),
|
||||
display_name: None,
|
||||
profile: None,
|
||||
job: None,
|
||||
workdir_attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// Real second-connection contention; the controller releases automatically on
|
||||
// failure, so a regression that blocks the Tokio thread cannot hang the suite.
|
||||
struct WriteLock {
|
||||
release: Option<std::sync::mpsc::Sender<()>>,
|
||||
controller: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl WriteLock {
|
||||
fn acquire(path: std::path::PathBuf) -> Self {
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
let (locked, ready) = std::sync::mpsc::channel();
|
||||
let controller = std::thread::spawn(move || {
|
||||
let connection = rusqlite::Connection::open(path).unwrap();
|
||||
connection.execute_batch("BEGIN IMMEDIATE").unwrap();
|
||||
locked.send(()).unwrap();
|
||||
let _ = released.recv_timeout(std::time::Duration::from_secs(3));
|
||||
connection.execute_batch("ROLLBACK").unwrap();
|
||||
});
|
||||
ready.recv().unwrap();
|
||||
Self {
|
||||
release: Some(release),
|
||||
controller: Some(controller),
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&mut self) {
|
||||
if let Some(release) = self.release.take() {
|
||||
let _ = release.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WriteLock {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
if let Some(controller) = self.controller.take() {
|
||||
controller.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_sql(service: &WorkerProjectionService) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
// With the second connection holding the write lock, a held ordering
|
||||
// gate identifies the accepted synchronous mutation, not a fake future.
|
||||
while service.ordering.try_lock().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("projection job did not enter synchronous SQL");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload()
|
||||
{
|
||||
use protocol::subscription::SubscriptionWorkerState;
|
||||
let (store, service, temp) = fixture().await;
|
||||
store
|
||||
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
|
||||
.unwrap();
|
||||
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
|
||||
assert_eq!(subscription.take_snapshot().len(), 1);
|
||||
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
|
||||
let publishing_service = service.clone();
|
||||
let publication = tokio::spawn(async move {
|
||||
publishing_service
|
||||
.seed_observation_async(
|
||||
"runtime-a",
|
||||
&observation(SubscriptionWorkerState::Idle),
|
||||
"2",
|
||||
)
|
||||
.await
|
||||
});
|
||||
wait_for_sql(&service).await;
|
||||
let started = std::time::Instant::now();
|
||||
let (control, controlled) = tokio::sync::oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
control.send("control progressed").unwrap();
|
||||
});
|
||||
let (_, control_result) = tokio::join!(
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)),
|
||||
controlled,
|
||||
);
|
||||
assert_eq!(control_result.unwrap(), "control progressed");
|
||||
assert!(
|
||||
started.elapsed() < std::time::Duration::from_secs(1),
|
||||
"SQLite blocked the current-thread executor"
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
lock.release();
|
||||
publication.await.unwrap().unwrap();
|
||||
let committed = subscription.recv().await.unwrap();
|
||||
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
|
||||
let snapshot = snapshot.take_snapshot();
|
||||
assert_eq!(
|
||||
committed.changes,
|
||||
vec![WorkerCatalogChange::Upsert(snapshot[0].clone())]
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot[0].observation.as_ref().unwrap().worker.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
service
|
||||
.seed_observation_async(
|
||||
"runtime-a",
|
||||
&observation(SubscriptionWorkerState::Running),
|
||||
"3",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Delayed readers receive the first transaction's payload, not a re-query
|
||||
// of mutable authority after a later commit.
|
||||
let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else {
|
||||
panic!("expected upsert");
|
||||
};
|
||||
assert_eq!(
|
||||
first.observation.as_ref().unwrap().worker.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
service
|
||||
.refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a"))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() {
|
||||
use crate::runtime_subscription::BrokerSubscriptionEvent;
|
||||
use protocol::subscription::SubscriptionWorkerState;
|
||||
let (store, service, temp) = fixture().await;
|
||||
store
|
||||
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
|
||||
.unwrap();
|
||||
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
|
||||
subscription.take_snapshot();
|
||||
let operations = service.replace_runtime("runtime-a").await;
|
||||
let old_operations = operations.clone();
|
||||
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
|
||||
let old_service = service.clone();
|
||||
let consumer = tokio::spawn(async move {
|
||||
old_service
|
||||
.apply_broker_event_owned_async(
|
||||
"runtime-a",
|
||||
BrokerSubscriptionEvent::Event {
|
||||
payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted {
|
||||
worker: observation(SubscriptionWorkerState::Running),
|
||||
},
|
||||
},
|
||||
operations,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
service
|
||||
.tasks
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("runtime-a")
|
||||
.unwrap()
|
||||
.consumer = Some(consumer);
|
||||
wait_for_sql(&service).await;
|
||||
let replacing_service = service.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
let _attach = replacing_service.attach_gate.lock().await;
|
||||
replacing_service.replace_runtime("runtime-a").await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
replacement.abort();
|
||||
assert!(replacement.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(
|
||||
old_operations.available_permits(),
|
||||
0,
|
||||
"cancelled drain lost accepted SQL ownership"
|
||||
);
|
||||
// A second replacement must still observe the old gate after cancellation.
|
||||
let replacing_service = service.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
let _attach = replacing_service.attach_gate.lock().await;
|
||||
let operations = replacing_service.replace_runtime("runtime-a").await;
|
||||
replacing_service
|
||||
.apply_broker_event_owned_async(
|
||||
"runtime-a",
|
||||
BrokerSubscriptionEvent::Snapshot {
|
||||
snapshot: protocol::subscription::SubscriptionSnapshot::Workers {
|
||||
workers: vec![observation(SubscriptionWorkerState::Idle)],
|
||||
},
|
||||
},
|
||||
operations,
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
lock.release();
|
||||
replacement.await.unwrap().unwrap();
|
||||
assert_eq!(old_operations.available_permits(), 1);
|
||||
let old_commit = subscription.recv().await.unwrap();
|
||||
let new_commit = subscription.recv().await.unwrap();
|
||||
for (commit, expected) in [
|
||||
(old_commit, SubscriptionWorkerState::Running),
|
||||
(new_commit, SubscriptionWorkerState::Idle),
|
||||
] {
|
||||
let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else {
|
||||
panic!("expected one committed upsert");
|
||||
};
|
||||
assert_eq!(record.observation.as_ref().unwrap().worker.state, expected);
|
||||
}
|
||||
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
|
||||
assert_eq!(
|
||||
snapshot.take_snapshot()[0]
|
||||
.observation
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.worker
|
||||
.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
service.shutdown_async().await;
|
||||
assert!(service.tasks.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn snapshot_registration_is_atomic_with_concurrent_publication() {
|
||||
let (store, service, _temp) = fixture().await;
|
||||
|
||||
@@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError {
|
||||
Replay,
|
||||
#[error("source Worker is not a current member of this Workspace Runtime catalog")]
|
||||
WorkerCatalogMembership,
|
||||
#[error("source proof authority is unavailable")]
|
||||
Unavailable,
|
||||
#[error("source proof authority failed: {0}")]
|
||||
Authority(String),
|
||||
}
|
||||
|
||||
impl From<crate::Error> for WorkerMutationSourceProofError {
|
||||
fn from(error: crate::Error) -> Self {
|
||||
match error {
|
||||
crate::Error::DbCapacityExceeded
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::Sqlite(_) => Self::Unavailable,
|
||||
other => Self::Authority(other.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn verify_worker_remove_source(
|
||||
api: &WorkspaceApi,
|
||||
proof: PresentedWorkerMutationSourceProof<'_>,
|
||||
@@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with(
|
||||
let trusted = store
|
||||
.get_workspace_runtime_binding(&config.workspace_id, &unverified.iss)
|
||||
.await
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?
|
||||
.map_err(WorkerMutationSourceProofError::from)?
|
||||
.filter(|record| record.revoked_at.is_none())
|
||||
.ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?;
|
||||
let public_key = trusted.public_key.as_str();
|
||||
@@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with(
|
||||
runtime_id: claims.iss.clone(),
|
||||
worker_id: claims.worker_id.clone(),
|
||||
};
|
||||
let member = store
|
||||
.get_worker_registry(&config.workspace_id, &worker)
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
let workspace_id = config.workspace_id.clone();
|
||||
let catalog_store = Arc::clone(store);
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
let member = catalog_store
|
||||
.get_worker_registry(&workspace_id, &worker)
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
if member.is_none() {
|
||||
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
|
||||
}
|
||||
store
|
||||
.require_current_worker_singleton_owner(&config.workspace_id, &worker)
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
catalog_store
|
||||
.require_current_worker_singleton_owner(&workspace_id, &worker)
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
|
||||
let consumed_at = chrono::Utc::now().to_rfc3339();
|
||||
let consumed = store
|
||||
@@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with(
|
||||
&consumed_at,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
if !consumed {
|
||||
return Err(WorkerMutationSourceProofError::Replay);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use worker_runtime::auth::{
|
||||
RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims,
|
||||
RuntimeRequestSourceExpectation, decode_runtime_request_source_claims,
|
||||
@@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError {
|
||||
pub(crate) exp: Option<i64>,
|
||||
pub(crate) now: Option<i64>,
|
||||
pub(crate) time_delta_seconds: Option<u64>,
|
||||
/// Operational unavailability is not a proof or principal rejection.
|
||||
pub(crate) availability: bool,
|
||||
}
|
||||
|
||||
impl RuntimeRequestProofError {
|
||||
pub fn is_availability_error(&self) -> bool {
|
||||
self.availability
|
||||
}
|
||||
|
||||
fn unavailable(mut self, reason: &'static str) -> Self {
|
||||
self.reason = reason;
|
||||
self.availability = true;
|
||||
self
|
||||
}
|
||||
|
||||
fn undecoded() -> Self {
|
||||
Self {
|
||||
reason: "invalid",
|
||||
@@ -47,6 +61,7 @@ impl RuntimeRequestProofError {
|
||||
exp: None,
|
||||
now: None,
|
||||
time_delta_seconds: None,
|
||||
availability: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +121,18 @@ impl RuntimeRequestProofError {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::Error> for RuntimeRequestProofError {
|
||||
fn from(error: crate::Error) -> Self {
|
||||
let reason = match error {
|
||||
crate::Error::DbCapacityExceeded => "db_capacity_exceeded",
|
||||
crate::Error::DbExecutorUnavailable => "db_executor_unavailable",
|
||||
crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown",
|
||||
_ => "database_authority_unavailable",
|
||||
};
|
||||
Self::undecoded().unavailable(reason)
|
||||
}
|
||||
}
|
||||
|
||||
/// Domain-separated fixed-length fingerprints, not raw request-derived IDs.
|
||||
/// Operators can correlate a known ID by applying this same function.
|
||||
pub fn diagnostic_id_hash(id: &str) -> String {
|
||||
@@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof(
|
||||
body_digest: &str,
|
||||
) -> Result<VerifiedRuntimeRequestSource, RuntimeRequestProofError> {
|
||||
verify_runtime_request_source_proof_with_store(
|
||||
api.store.as_ref(),
|
||||
Arc::clone(&api.store),
|
||||
&api.config,
|
||||
proof,
|
||||
workspace_id,
|
||||
@@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn verify_runtime_request_source_proof_with_store(
|
||||
store: &dyn ControlPlaneStore,
|
||||
store: Arc<dyn ControlPlaneStore>,
|
||||
config: &ServerConfig,
|
||||
proof: &str,
|
||||
workspace_id: &str,
|
||||
@@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
store: &dyn ControlPlaneStore,
|
||||
store: Arc<dyn ControlPlaneStore>,
|
||||
config: &ServerConfig,
|
||||
proof: &str,
|
||||
workspace_id: &str,
|
||||
@@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
let unverified = decode_runtime_request_source_claims(proof)
|
||||
.map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?;
|
||||
let diagnostic = RuntimeRequestProofError::decoded(&unverified);
|
||||
let audience = remote_audience(config, workspace_id)
|
||||
.map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?;
|
||||
let audience = remote_audience(config, workspace_id).map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.unavailable("backend_audience_unavailable")
|
||||
})?;
|
||||
let trusted = store
|
||||
.get_workspace_runtime_binding(workspace_id, &unverified.iss)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("runtime_trust_authority_unavailable")
|
||||
.unavailable("runtime_trust_authority_unavailable")
|
||||
})?
|
||||
.filter(|record| record.revoked_at.is_none())
|
||||
.ok_or_else(|| {
|
||||
@@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
&consumed_at,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))?
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.unavailable("replay_authority_unavailable")
|
||||
})?
|
||||
{
|
||||
return Err(diagnostic.reason("replay"));
|
||||
}
|
||||
@@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
runtime_id: claims.iss.clone(),
|
||||
worker_id: worker_id.to_owned(),
|
||||
};
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let executor = store.db_executor();
|
||||
let stage_diagnostic = diagnostic.clone();
|
||||
// Only synchronous catalog authority lives in this closure. The trust
|
||||
// and JTI async wrappers above already offload through the executor.
|
||||
executor
|
||||
.run(move || {
|
||||
let member = store
|
||||
.get_worker_registry(workspace_id, &worker)
|
||||
.get_worker_registry(&workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.reason("worker_catalog_authority_unavailable")
|
||||
.unavailable("worker_catalog_authority_unavailable")
|
||||
})?;
|
||||
let reserved = store
|
||||
.has_active_worker_create_reservation(workspace_id, &worker)
|
||||
.has_active_worker_create_reservation(&workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.reason("worker_reservation_authority_unavailable")
|
||||
.unavailable("worker_reservation_authority_unavailable")
|
||||
})?;
|
||||
if member.is_none() && !reserved {
|
||||
return Err(diagnostic.reason("worker_catalog_membership"));
|
||||
return Err(stage_diagnostic.reason("worker_catalog_membership"));
|
||||
}
|
||||
store
|
||||
.require_current_worker_singleton_owner(workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.require_current_worker_singleton_owner(&workspace_id, &worker)
|
||||
.map_err(|error| {
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbCapacityExceeded
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::Sqlite(_)
|
||||
) {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.unavailable("worker_singleton_authority_unavailable")
|
||||
} else {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.reason("worker_singleton_authority_denied")
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|error: RuntimeRequestProofError| {
|
||||
if error.verification == "unverified" && error.is_availability_error() {
|
||||
diagnostic.clone().unavailable(error.reason)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
Ok(VerifiedRuntimeRequestSource {
|
||||
@@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
authentication_binding_id: trusted.binding_id,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::{
|
||||
AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode,
|
||||
WorkspaceRuntimeBinding,
|
||||
};
|
||||
use std::time::Duration;
|
||||
use worker_runtime::auth::{
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner,
|
||||
request_body_digest,
|
||||
};
|
||||
|
||||
const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa";
|
||||
const NOW: i64 = 1_700_000_000;
|
||||
const PATH: &str = "/api/runtime/v1/resources/fetch";
|
||||
|
||||
// rusqlite's busy handler accepts a function pointer, not a closure. Each
|
||||
// fixture owns a dedicated single-worker executor, so this thread-local
|
||||
// handshake is isolated from all parallel tests and product connections.
|
||||
thread_local! {
|
||||
static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell<Option<(
|
||||
tokio::sync::oneshot::Sender<()>,
|
||||
std::sync::mpsc::Receiver<()>,
|
||||
)>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn acknowledge_sqlite_contention(_attempt: i32) -> bool {
|
||||
SQLITE_BUSY_HANDSHAKE.with(|slot| {
|
||||
let Some((blocked, retry)) = slot.borrow_mut().take() else {
|
||||
return false;
|
||||
};
|
||||
let _ = blocked.send(());
|
||||
// The test releases SQLite's writer lock before allowing retry.
|
||||
// Dropping the sender on panic also releases this worker promptly.
|
||||
retry.recv_timeout(Duration::from_secs(3)).is_ok()
|
||||
})
|
||||
}
|
||||
|
||||
async fn fixture(
|
||||
root: &std::path::Path,
|
||||
) -> (Arc<SqliteWorkspaceStore>, ServerConfig, String, String) {
|
||||
let db_path = root.join("auth.db");
|
||||
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open(&db_path)
|
||||
.unwrap()
|
||||
.with_db_executor(executor),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-auth".into(),
|
||||
kind: "user".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
let workspace = WorkspaceRecord {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
owner_account_id: "account-auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
};
|
||||
store.upsert_workspace(&workspace).await.unwrap();
|
||||
let config = ServerConfig {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
workspace_display_name: "Auth".into(),
|
||||
workspace_created_at: "1".into(),
|
||||
workspace_execution_root: root.to_owned(),
|
||||
database_path: db_path,
|
||||
data_root: root.to_owned(),
|
||||
embedded_runtime_store_root: root.join("runtime"),
|
||||
static_assets_dir: None,
|
||||
auth: crate::server::AuthConfig::Passkey {
|
||||
rp_id: "localhost".into(),
|
||||
origin: "http://localhost".into(),
|
||||
public_base_url: "http://localhost".into(),
|
||||
cookie_name: "session".into(),
|
||||
},
|
||||
max_records: 10,
|
||||
repositories: vec![],
|
||||
runtime_event_sources: vec![],
|
||||
remote_runtime_sources: vec![],
|
||||
backend_base_url: Some("server-auth".into()),
|
||||
};
|
||||
let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap();
|
||||
store
|
||||
.upsert_workspace_runtime_binding_record(
|
||||
WorkspaceRuntimeBinding {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
runtime_id: "runtime-auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
base_url: "http://127.0.0.1:1".into(),
|
||||
public_key: identity.public_key.clone(),
|
||||
public_key_fingerprint: String::new(),
|
||||
binding_id: String::new(),
|
||||
authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
revoked_at: None,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let binding = ControlPlaneStore::get_workspace_runtime_binding(
|
||||
store.as_ref(),
|
||||
WORKSPACE,
|
||||
"runtime-auth",
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let proof = RuntimeRequestSourceSigner::from_identity(&identity)
|
||||
.issue(
|
||||
"server-auth",
|
||||
WORKSPACE,
|
||||
None,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
b"{}",
|
||||
NOW,
|
||||
30,
|
||||
)
|
||||
.unwrap();
|
||||
(store, config, proof, binding.binding_id)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (store, config, proof, binding_id) = fixture(root.path()).await;
|
||||
// A real independent SQLite writer prevents JTI insertion while trust
|
||||
// reads remain possible. Rollback on Drop also releases this on panic.
|
||||
let mut writer = rusqlite::Connection::open(&config.database_path).unwrap();
|
||||
let lock = writer
|
||||
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let (blocked, contention_reached) = tokio::sync::oneshot::channel();
|
||||
let (retry, retry_rx) = std::sync::mpsc::channel();
|
||||
let setup_store = store.clone();
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || -> crate::Result<()> {
|
||||
SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx)));
|
||||
setup_store.with_conn(|conn| {
|
||||
conn.busy_handler(Some(acknowledge_sqlite_contention))?;
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let request_store = store.clone();
|
||||
let request_config = config.clone();
|
||||
let request_proof = proof.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
verify_runtime_request_source_proof_with_clock(
|
||||
request_store,
|
||||
&request_config,
|
||||
&request_proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), contention_reached)
|
||||
.await
|
||||
.expect(
|
||||
"SQLite JTI persistence must report real writer contention without blocking Tokio",
|
||||
)
|
||||
.unwrap();
|
||||
// The busy callback acknowledges an actual SQLite write blocked by the
|
||||
// independent writer, then holds retry until explicitly released.
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"JTI must remain pending while SQLite writer is locked"
|
||||
);
|
||||
let control = tokio::spawn(async { "control-responsive" });
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), control)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
"control-responsive"
|
||||
);
|
||||
// This timer is the behavior being tested, not a guessed wait for SQL.
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
tokio::time::sleep(Duration::from_millis(1)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"timer must advance while JTI remains blocked"
|
||||
);
|
||||
lock.rollback().unwrap();
|
||||
retry.send(()).unwrap();
|
||||
let source = tokio::time::timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(source.runtime_id, "runtime-auth");
|
||||
assert_eq!(source.authentication_binding_id, binding_id);
|
||||
let replay = verify_runtime_request_source_proof_with_clock(
|
||||
store,
|
||||
&config,
|
||||
&proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(replay.reason, "replay");
|
||||
assert!(!replay.is_availability_error());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_database_is_availability_failure_not_invalid_proof() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (store, config, proof, _) = fixture(root.path()).await;
|
||||
// Fill the isolated executor with a blocked job and queued jobs. No
|
||||
// scheduling sleeps: the worker explicitly acknowledges its start.
|
||||
let executor = store.db_executor();
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker_executor = executor.clone();
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_executor
|
||||
.run(move || -> crate::Result<()> {
|
||||
let _ = started.send(());
|
||||
release_rx.recv().unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) }));
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
verify_runtime_request_source_proof_with_clock(
|
||||
store,
|
||||
&config,
|
||||
&proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
// Release before assertions so a failed contract never strands a worker.
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap().unwrap();
|
||||
queued.await.unwrap();
|
||||
let error = outcome
|
||||
.expect("capacity rejection must not wait for queued SQL")
|
||||
.unwrap_err();
|
||||
assert_eq!(error.reason, "runtime_trust_authority_unavailable");
|
||||
assert!(error.is_availability_error());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost {
|
||||
impl WorkspaceDriveHost {
|
||||
pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result<Self> {
|
||||
let storage = storage.clone();
|
||||
let registration = tokio::task::spawn_blocking(move || Drive::register(&storage))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))?
|
||||
.map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?;
|
||||
let registration = crate::db_executor::DbExecutor::shared()
|
||||
.run(move || {
|
||||
Drive::register(&storage).map_err(|error| {
|
||||
Error::Store(format!("failed to register Drive storage: {error}"))
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
Ok(Self {
|
||||
registration,
|
||||
purged: AtomicBool::new(false),
|
||||
@@ -80,7 +83,8 @@ impl WorkspaceDriveHost {
|
||||
}
|
||||
let storage = storage.clone();
|
||||
let registration = self.registration.clone();
|
||||
let drive = tokio::task::spawn_blocking(move || {
|
||||
let drive = crate::db_executor::DbExecutor::shared()
|
||||
.run(move || {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain IMMEDIATE transactions lock the attached server authority
|
||||
// as well as Drive metadata. Deletion reservation and cached-handle
|
||||
@@ -93,8 +97,7 @@ impl WorkspaceDriveHost {
|
||||
)
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??;
|
||||
.await?;
|
||||
*cached = Some(drive.clone());
|
||||
Ok(drive)
|
||||
}
|
||||
@@ -109,9 +112,9 @@ impl WorkspaceDriveHost {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root, authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from))
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || drive.fence().map_err(Error::from))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))?
|
||||
}
|
||||
|
||||
pub(crate) async fn purge(
|
||||
@@ -124,7 +127,8 @@ impl WorkspaceDriveHost {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root.clone(), authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || -> Result<()> {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
|
||||
drive.purge()?;
|
||||
@@ -136,8 +140,7 @@ impl WorkspaceDriveHost {
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??;
|
||||
.await?;
|
||||
// A later signing-material/finalization failure can retry after the
|
||||
// feature DB has been closed and removed by FeatureStorage.delete.
|
||||
self.purged.store(true, Ordering::Release);
|
||||
|
||||
@@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription<S>(
|
||||
worker_id,
|
||||
runtime_id: Some(runtime_id),
|
||||
} => {
|
||||
let connection = match WorkspaceWorker::resolve(
|
||||
let connection = match WorkspaceWorker::resolve_async(
|
||||
&api,
|
||||
&runtime_id,
|
||||
worker_id.as_str(),
|
||||
) {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(worker) => worker
|
||||
.connect_protocol(&context)
|
||||
.await
|
||||
@@ -366,7 +368,8 @@ async fn run_workspace_workers(
|
||||
) {
|
||||
let Ok(mut projection) = api
|
||||
.worker_projection
|
||||
.subscribe_ordered(api.config.max_records.max(1))
|
||||
.subscribe_ordered_async(api.config.max_records.max(1))
|
||||
.await
|
||||
else {
|
||||
let _ = send_rejected(
|
||||
&outbound,
|
||||
@@ -377,11 +380,22 @@ async fn run_workspace_workers(
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let mut workers = projection
|
||||
.take_snapshot()
|
||||
.into_iter()
|
||||
.filter_map(|record| project_registry_worker(&api, record))
|
||||
.collect::<Vec<_>>();
|
||||
let Ok(mut workers) = project_registry_workers_async(
|
||||
api.store.clone(),
|
||||
api.config.workspace_id.clone(),
|
||||
projection.take_snapshot(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
let _ = send_rejected(
|
||||
&outbound,
|
||||
request_id,
|
||||
SubscriptionRejectionCode::Internal,
|
||||
"failed to enrich durable Worker projection".to_string(),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
sort_workers(&mut workers);
|
||||
if send_frame(
|
||||
&outbound,
|
||||
@@ -406,7 +420,29 @@ async fn run_workspace_workers(
|
||||
for change in event.changes {
|
||||
let payload = match change {
|
||||
crate::worker_projection::WorkerProjectionChange::Upsert(record) => {
|
||||
let Some(worker) = project_registry_worker(&api, record) else {
|
||||
let workers = project_registry_workers_async(
|
||||
api.store.clone(),
|
||||
api.config.workspace_id.clone(),
|
||||
vec![record],
|
||||
)
|
||||
.await;
|
||||
let Ok(mut workers) = workers else {
|
||||
// Admission failure cannot silently lose a live
|
||||
// update: terminate and require a fresh snapshot.
|
||||
let _ = send_frame(
|
||||
&outbound,
|
||||
SubscriptionFrame::new(SubscriptionFramePayload::Event(
|
||||
SubscriptionEvent::SubscriptionClosed {
|
||||
subscription_id: subscription_id.clone(),
|
||||
code: SubscriptionTerminationCode::Lagged,
|
||||
message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(),
|
||||
},
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let Some(worker) = workers.pop() else {
|
||||
continue;
|
||||
};
|
||||
SubscriptionEventPayload::WorkerUpserted { worker }
|
||||
@@ -450,8 +486,28 @@ async fn run_workspace_workers(
|
||||
}
|
||||
}
|
||||
|
||||
async fn project_registry_workers_async(
|
||||
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
|
||||
workspace_id: String,
|
||||
records: Vec<crate::store::WorkerRegistryProjectionRecord>,
|
||||
) -> crate::Result<Vec<SubscriptionWorker>> {
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
// Keep the committed Worker payload; only the existing Workspace-scoped
|
||||
// workdir/history/repository enrichment reads occur inside this job.
|
||||
Ok(records
|
||||
.into_iter()
|
||||
.filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record))
|
||||
.collect())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
// Synchronous helpers stay synchronous; they run only inside the job above.
|
||||
fn project_registry_worker(
|
||||
api: &WorkspaceApi,
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
record: crate::store::WorkerRegistryProjectionRecord,
|
||||
) -> Option<SubscriptionWorker> {
|
||||
let runtime_id = record.registry.worker.runtime_id.clone();
|
||||
@@ -491,29 +547,25 @@ fn project_registry_worker(
|
||||
attempt_id: job.attempt_id,
|
||||
purpose: job.purpose,
|
||||
});
|
||||
if !project_working_directory(api, &record.registry.worker, &mut worker) {
|
||||
if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) {
|
||||
return None;
|
||||
}
|
||||
Some(worker)
|
||||
}
|
||||
|
||||
fn project_working_directory(
|
||||
api: &WorkspaceApi,
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
worker_ref: &RuntimeWorkerRef,
|
||||
worker: &mut SubscriptionWorker,
|
||||
) -> bool {
|
||||
let Ok(links) = api
|
||||
.store
|
||||
.list_worker_workdir_links(&api.config.workspace_id, worker_ref)
|
||||
else {
|
||||
let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else {
|
||||
return false;
|
||||
};
|
||||
if !links.is_empty() {
|
||||
let mut attachments = Vec::with_capacity(links.len());
|
||||
for link in links {
|
||||
let Ok(Some(workdir)) = api
|
||||
.store
|
||||
.get_workdir_registry(&api.config.workspace_id, &link.workdir_id)
|
||||
let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -533,9 +585,8 @@ fn project_working_directory(
|
||||
}
|
||||
worker.workdir_attachments = attachments;
|
||||
} else {
|
||||
let Ok(has_link_history) = api
|
||||
.store
|
||||
.worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref)
|
||||
let Ok(has_link_history) =
|
||||
store.worker_workdir_link_history_exists(workspace_id, worker_ref)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -543,13 +594,17 @@ fn project_working_directory(
|
||||
worker.workdir_attachments.clear();
|
||||
}
|
||||
}
|
||||
project_repository_keys(api, worker)
|
||||
project_repository_keys(store, workspace_id, worker)
|
||||
}
|
||||
|
||||
fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool {
|
||||
fn project_repository_keys(
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
worker: &mut SubscriptionWorker,
|
||||
) -> bool {
|
||||
project_repository_keys_with(worker, |repository_id| {
|
||||
api.store
|
||||
.get_repository(&api.config.workspace_id, repository_id)
|
||||
store
|
||||
.get_repository(workspace_id, repository_id)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|repository| repository.repository_key)
|
||||
@@ -634,9 +689,8 @@ mod tests {
|
||||
protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||
let mut worker = SubscriptionWorker {
|
||||
fn external_worker() -> SubscriptionWorker {
|
||||
SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: worker_id(),
|
||||
runtime_id: Some("runtime-1".to_string()),
|
||||
@@ -660,8 +714,12 @@ mod tests {
|
||||
.unwrap(),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||
let mut worker = external_worker();
|
||||
assert!(project_repository_keys_with(&mut worker, |_| {
|
||||
panic!("External Workdir must not require Repository projection")
|
||||
}));
|
||||
@@ -673,6 +731,109 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository()
|
||||
{
|
||||
use crate::store::{
|
||||
AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord,
|
||||
WorkspaceRecord,
|
||||
};
|
||||
let store =
|
||||
std::sync::Arc::new(SqliteWorkspaceStore::in_memory().unwrap().with_db_executor(
|
||||
crate::db_executor::DbExecutor::new(Default::default()).unwrap(),
|
||||
));
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".into(),
|
||||
kind: "user".into(),
|
||||
handle: "owner".into(),
|
||||
display_name: "Owner".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-1".into(),
|
||||
owner_account_id: "owner".into(),
|
||||
display_name: "Enrichment".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1");
|
||||
store
|
||||
.upsert_worker_registry(&WorkerRegistryRecord {
|
||||
workspace_id: "workspace-1".into(),
|
||||
worker: worker_ref.clone(),
|
||||
display_name: "Durable name".into(),
|
||||
profile: Some("builtin:coder".into()),
|
||||
retention_state: "normal".into(),
|
||||
transcript_ref: None,
|
||||
session_ref: None,
|
||||
summary_ref: None,
|
||||
diagnostics_ref: None,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2")
|
||||
.unwrap();
|
||||
let committed = store
|
||||
.worker_registry_projection("workspace-1", &worker_ref)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
// Change authority after capturing the payload, as can happen while async
|
||||
// enrichment is queued. It must not re-query and replace the committed Worker.
|
||||
let mut later = external_worker();
|
||||
later.state = protocol::subscription::SubscriptionWorkerState::Running;
|
||||
store
|
||||
.apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3")
|
||||
.unwrap();
|
||||
let projected = project_registry_workers_async(
|
||||
store.clone(),
|
||||
"workspace-1".into(),
|
||||
vec![committed.clone()],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(projected.len(), 1);
|
||||
let worker = &projected[0];
|
||||
assert_eq!(
|
||||
worker.state,
|
||||
protocol::subscription::SubscriptionWorkerState::Idle
|
||||
);
|
||||
assert_eq!(
|
||||
worker.availability,
|
||||
protocol::subscription::SubscriptionWorkerAvailability::Observed
|
||||
);
|
||||
assert_eq!(worker.display_name.as_deref(), Some("Durable name"));
|
||||
assert_eq!(worker.profile.as_deref(), Some("builtin:coder"));
|
||||
assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1"));
|
||||
assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1"));
|
||||
assert_eq!(
|
||||
worker.workdir_attachments,
|
||||
external_worker().workdir_attachments
|
||||
);
|
||||
let mut unresolved = committed;
|
||||
unresolved
|
||||
.observation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.worker
|
||||
.workdir_attachments[0]
|
||||
.repository_id = Some("missing-repository".into());
|
||||
assert!(
|
||||
project_registry_workers_async(store, "workspace-1".into(), vec![unresolved])
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_with_events(events: mpsc::Receiver<protocol::Event>) -> SubscriptionResponse {
|
||||
let (control_outbound, mut control_receiver) = mpsc::channel(1);
|
||||
let (protocol_outbound, _protocol_receiver) = mpsc::channel(1);
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# Backend bounded synchronous execution
|
||||
|
||||
## Ownership and admission
|
||||
|
||||
`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner.
|
||||
Use `executor.run(move || synchronous_operation(...)).await` with owned input and
|
||||
services. The closure owns the **whole** synchronous operation, including connection
|
||||
acquisition, mutex wait, the existing SQL transaction, commit, and ordered local
|
||||
publication. Do not split source authorization, receipts, CAS, audit, or saved
|
||||
results around an await. Do not change connection ownership or introduce a read pool.
|
||||
|
||||
Production stores share one DB worker and a queue of 32 jobs. Admission uses a
|
||||
nonblocking queue send and a separate actual-job budget (worker count plus queue
|
||||
capacity); rejection means the operation was **not accepted**. HTTP maps capacity
|
||||
and executor unavailability to 503, not authentication failure or deletion conflict.
|
||||
Tests can inject a dedicated executor with explicit limits, without process-global
|
||||
settings or environment variables.
|
||||
|
||||
Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter:
|
||||
|
||||
- does not interrupt, resubmit, or retry SQL;
|
||||
- does not return its queue/job slot before execution and discarded-result cleanup;
|
||||
- does not release the request lifecycle lease before actual completion.
|
||||
|
||||
Synchronous post-commit continuations inherit the admitted lifecycle source when
|
||||
submitting another bounded job. This is not implicit inheritance by detached Tokio
|
||||
tasks. An operation panic has an unknown outcome even if SQL already committed;
|
||||
return a structured unknown-outcome error and diagnostic, never retry the mutation.
|
||||
A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all
|
||||
jobs accepted before its barrier is first polled and waits for actual cleanup; it
|
||||
is not a queue close and does not include later admissions. Lifecycle callers must
|
||||
stop their producers/admission before destroying resources.
|
||||
|
||||
## Runtime/network boundaries
|
||||
|
||||
A synchronous Runtime workflow must not monopolize the sole DB worker while
|
||||
waiting for callbacks that themselves need DB authority. The Server's separate
|
||||
finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job
|
||||
ownership mechanism, not an unbounded `spawn_blocking` queue.
|
||||
|
||||
Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can
|
||||
use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an
|
||||
owned workflow needs async resource locks, `run_host_workflow` polls it on a Host
|
||||
worker using the original Tokio handle; do not build a replacement runtime for
|
||||
cached services and do not submit to and await the Host lane from inside itself.
|
||||
Control/session guards needed by an accepted mutation belong to that actual job.
|
||||
|
||||
Backend Job cleanup is admitted before its durable cleanup claim. Rejection leaves
|
||||
Pending/Failed ownership recoverable instead of abandoning a Running claim. Ticket
|
||||
checker dispatch and timeout processing use bounded admission; overload leaves the
|
||||
saved reservation/deadline diagnosable. They are not automatically retried after an
|
||||
unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof
|
||||
of model processing.
|
||||
|
||||
Remote subscription and Workdir authorization are async callbacks: their complete
|
||||
live binding check and fresh request signing are offloaded together. A queue error
|
||||
must not fall back to a configured bearer token. Synchronous CLI/migration callers
|
||||
and existing synchronous Runtime adapters retain their explicit blocking boundary.
|
||||
|
||||
## Caller recipe (including assignment APIs)
|
||||
|
||||
```rust
|
||||
let store = api.store.clone();
|
||||
let workspace_id = api.workspace_id().to_owned();
|
||||
let worker = worker.clone();
|
||||
let record = api.store.db_executor().run(move || {
|
||||
store.get_worker_registry(&workspace_id, &worker)
|
||||
}).await?;
|
||||
```
|
||||
|
||||
This is an illustration, not a new authority API. Assignment delivery scan,
|
||||
reserve, prepare, acknowledgement and start authorization must keep their existing
|
||||
whole transaction helpers. Additional assignment diagnostics, answer mutations,
|
||||
and Ticket queries use the same owned caller boundary; do not add a second ledger
|
||||
or reconstruct processing evidence from Worker liveness.
|
||||
|
||||
## Regression evidence
|
||||
|
||||
- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread
|
||||
timer/control task progresses; accepted/cancelled queue bounds; rollback; panic
|
||||
before/after commit; discarded reply cleanup and retained authority; child jobs.
|
||||
- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope
|
||||
opening/mutation under a separate SQLite connection's write lock; overload;
|
||||
cancelled HTTP jobs and Workspace deletion drain.
|
||||
- `worker_projection::tests`: real SQL contention, commit/payload publication,
|
||||
ordered snapshot registration, and replacement draining a cancelled consumer's
|
||||
accepted SQL before applying a new Runtime snapshot.
|
||||
- Auth/proof and Runtime subscription tests protect availability classification,
|
||||
replay rejection, and no bearer fallback.
|
||||
- `workdir`'s `http-client` tests protect awaited fresh authorization and exact
|
||||
request claims before any HTTP send.
|
||||
|
||||
These fixtures use temporary authority data and observable synchronization, not
|
||||
live databases or a fixed sleep to guess completion. This change does not claim a
|
||||
latency benchmark or a measured production incident.
|
||||
Reference in New Issue
Block a user