merge: compose T748 bounded execution with T743 public callers

This commit is contained in:
2026-10-12 09:00:39 +09:00
62 changed files with 11957 additions and 4223 deletions
+145 -16
View File
@@ -432,16 +432,23 @@ mod client {
use super::*;
use crate::{Workdir, WorkdirSession};
/// Provides a fresh bearer token for each Runtime request. Backend
/// implementations can mint short-lived capability tokens without making a
/// Worker-bound session expire with the token used to open it.
/// An owned, Send authorization future that can offload synchronous authority
/// work without borrowing request data or the authorization provider.
pub type WorkdirHttpAuthorizationFuture = std::pin::Pin<
Box<dyn std::future::Future<Output = Result<String, WorkdirError>> + Send + 'static>,
>;
/// Provides a fresh bearer token asynchronously for each Runtime request.
/// Backend implementations can mint short-lived capability tokens without
/// making a Worker-bound session expire with the token used to open it.
/// Blocking DB/signing work must be offloaded by the implementation.
pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync {
fn bearer_token(
&self,
method: &str,
path_and_query: &str,
body: &[u8],
) -> Result<String, WorkdirError>;
method: String,
path_and_query: String,
body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture;
}
struct FixedBearerToken(Arc<str>);
@@ -455,11 +462,12 @@ mod client {
impl WorkdirHttpAuthorization for FixedBearerToken {
fn bearer_token(
&self,
_method: &str,
_path_and_query: &str,
_body: &[u8],
) -> Result<String, WorkdirError> {
Ok(self.0.to_string())
_method: String,
_path_and_query: String,
_body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture {
let token = self.0.clone();
Box::pin(async move { Ok(token.to_string()) })
}
}
@@ -509,7 +517,9 @@ mod client {
)?;
let body = serde_json::to_vec(&request)
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
let token = authorization.bearer_token("POST", url.path(), &body)?;
let token = authorization
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
.await?;
let response = client
.post(url)
.bearer_auth(token)
@@ -560,7 +570,10 @@ mod client {
let operation = WorkdirSessionOperationRequest { operation };
let body = serde_json::to_vec(&operation)
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
let token = self.authorization.bearer_token("POST", url.path(), &body)?;
let token = self
.authorization
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
.await?;
let response = self
.client
.post(url)
@@ -864,7 +877,10 @@ mod client {
&self.base_url,
&["v1", "workdir-sessions", self.session_id.as_str()],
)?;
let token = self.authorization.bearer_token("DELETE", url.path(), &[])?;
let token = self
.authorization
.bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new())
.await?;
let response = self
.client
.delete(url)
@@ -918,11 +934,124 @@ mod client {
WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string())
}
#[cfg(test)]
mod authorization_tests {
use super::*;
use std::sync::Mutex;
#[derive(Debug, Default)]
struct RejectAuthorization {
requests: Arc<Mutex<Vec<(String, String, Vec<u8>)>>>,
}
impl WorkdirHttpAuthorization for RejectAuthorization {
fn bearer_token(
&self,
method: String,
path_and_query: String,
body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture {
let requests = self.requests.clone();
Box::pin(async move {
tokio::task::yield_now().await;
let mut requests = requests.lock().unwrap();
requests.push((method, path_and_query, body));
Err(WorkdirError::Unavailable(format!(
"authorization rejected request {}",
requests.len()
)))
})
}
}
#[tokio::test]
async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let base_url: Url = format!("http://{}", listener.local_addr().unwrap())
.parse()
.unwrap();
let client = Client::builder().no_proxy().build().unwrap();
let authorization = Arc::new(RejectAuthorization::default());
let workdir = Workdir::new("w");
let open_request = OpenWorkdirSessionRequest {
owner_worker_id: Some("worker".to_string()),
};
let error = RemoteWorkdirSession::open_with_authorization(
client.clone(),
base_url.clone(),
authorization.clone(),
workdir.id().clone(),
open_request.clone(),
)
.await
.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == "authorization rejected request 1"
));
// Seed an already-open session to exercise the public operation and
// close boundaries independently of the rejected open request.
let remote = RemoteWorkdirSession {
client,
base_url,
authorization: authorization.clone(),
workdir,
session_id: WorkdirSessionId::new("s").unwrap(),
capabilities: WorkdirSessionCapabilities::EMPTY,
closed: AtomicBool::new(false),
};
let stat = StatRequest {
path: crate::WorkdirPath::new("file").unwrap(),
};
let error = remote.stat(stat.clone()).await.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == "authorization rejected request 2"
));
let error = remote.close().await.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == "authorization rejected request 3"
));
assert_eq!(
*authorization.requests.lock().unwrap(),
vec![
(
"POST".to_string(),
"/v1/working-directories/w/sessions".to_string(),
serde_json::to_vec(&open_request).unwrap(),
),
(
"POST".to_string(),
"/v1/workdir-sessions/s/operations".to_string(),
serde_json::to_vec(&WorkdirSessionOperationRequest {
operation: WorkdirSessionOperation::Stat(stat),
})
.unwrap(),
),
(
"DELETE".to_string(),
"/v1/workdir-sessions/s".to_string(),
Vec::new(),
),
]
);
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
}
}
pub use self::RemoteWorkdirSession as ClientSession;
}
#[cfg(feature = "http-client")]
pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization};
pub use client::{
ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
};
#[cfg(test)]
mod tests {
@@ -147,7 +147,7 @@ async fn assignment_delivery_response_loss_returns_immutable_payload_after_progr
.update_ticket_assignment_tracking("w", "tracking", &update, &source)
.unwrap();
drop(store);
let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
assert_eq!(
store
.list_pending_ticket_assignment_deliveries("w", 100)
@@ -208,7 +208,7 @@ async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt()
let path = directory.path().join("workspace.db");
let input = pending.input_event_id.clone();
std::thread::spawn(move || {
let store = SqliteWorkspaceStore::open(path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
barrier.wait();
store
.reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source())
@@ -568,7 +568,7 @@ async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_co
"original rendered prompt"
);
drop(store);
let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
assert_eq!(
store
.prepare_ticket_assignment_delivery_content(
@@ -634,7 +634,7 @@ fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_l
let operations = stored_fixture_rows(&conn, "ticket_assignment_operations");
let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history");
drop(conn);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let pending = store
.list_pending_ticket_assignment_deliveries("w", 100)
.unwrap();
@@ -684,7 +684,7 @@ async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once
let barrier = barrier.clone();
let path = directory.path().join("workspace.db");
std::thread::spawn(move || {
let store = SqliteWorkspaceStore::open(path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
barrier.wait();
delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source())
.unwrap()
@@ -716,8 +716,8 @@ fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() {
let conn = tracking_schema91_fixture(&path);
migrate_ticket_tracking_v91_to_v92(&conn).unwrap();
drop(conn);
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
let fresh = SqliteWorkspaceStore::in_memory().unwrap();
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
let objects = |store: &SqliteWorkspaceStore| {
store.with_conn(|conn| {
let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?;
@@ -741,7 +741,7 @@ async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() {
.unwrap();
drop(store);
assert!(
matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update"))
matches!(SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update"))
);
}
@@ -78,7 +78,7 @@ async fn tracking_store_every_replay_fences_current_singleton_and_authenticated_
);
// Model the authority change after authenticated source capture, before
// BEGIN IMMEDIATE on a second connection. Receipt presence cannot bypass it.
let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
second.with_conn_mut(|conn| {
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
match change {
@@ -233,7 +233,7 @@ async fn tracking_store_same_principal_reissue_and_exact_retries_keep_original_r
.is_err()
);
drop(store);
let reopened = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
let reopened = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
assert_eq!(
reopened
.get_ticket_assignment_tracking("w", "tracking", "a")
@@ -441,7 +441,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
.collect::<Vec<_>>(),
vec![92, 93]
);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store
.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?, 93);
@@ -525,7 +525,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
.list_ticket_assignment_tracking_history("w", "pending", "a-pending")
.unwrap();
drop(store);
let reopened = SqliteWorkspaceStore::open(&path).unwrap();
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
assert_eq!(
reopened
.list_ticket_assignment_tracking_history("w", "pending", "a-pending")
@@ -550,7 +550,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
let releases_before = stored_fixture_rows(&conn, "ticket_assignment_work_releases");
drop(conn);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let backend = ticket::SqliteTicketBackend::open_verified(&path, "w").unwrap();
let reopen = || {
let observed = backend.show("historical-done".into()).unwrap();
@@ -645,7 +645,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
);
// A second observer holds the same Assign baseline. Only the first
// explicit writer may consume it; BEGIN IMMEDIATE serializes the CAS.
let second = SqliteWorkspaceStore::open(&path).unwrap();
let second = SqliteWorkspaceStore::open_for_test(&path).unwrap();
assert_eq!(
second
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
@@ -740,7 +740,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
}).unwrap();
drop(second);
drop(store);
let reopened_store = SqliteWorkspaceStore::open(&path).unwrap();
let reopened_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
assert_eq!(
reopened_store
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
@@ -796,7 +796,7 @@ fn tracking_store_schema91_missing_assign_audit_remains_unknown_without_cas_base
)
.unwrap();
drop(conn);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let unknown = store
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
.unwrap()
@@ -1068,7 +1068,7 @@ async fn orchestrator_assignment_all_runtime_retries_fence_singleton_and_binding
&source,
)
.unwrap();
let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
second.with_conn_mut(|conn| {let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;match change {
"replace" => {tx.execute("UPDATE worker_singleton_owners SET worker_id='replacement',generation=2 WHERE workspace_id='w'",[])?;},
"revoke" => {tx.execute("UPDATE workspace_runtime_bindings SET revoked_at='revoked' WHERE workspace_id='w'",[])?;},
+198 -4
View File
@@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option<String> {
})
}
pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
store: &S,
pub async fn resolve_request_actor(
store: std::sync::Arc<dyn ControlPlaneStore>,
headers: &HeaderMap,
cookie_name: &str,
) -> Result<Option<RequestActor>> {
if let Some(token) = parse_bearer(headers) {
let bearer = parse_bearer(headers);
let session = parse_cookie(headers, cookie_name);
if bearer.is_none() && session.is_none() {
return Ok(None);
}
let executor = store.db_executor();
executor
.run(move || resolve_request_actor_sync(store.as_ref(), bearer, session))
.await
}
// One owned synchronous stage includes token usage marking and the user read.
// Do not call async store wrappers here: they use the same executor themselves.
fn resolve_request_actor_sync(
store: &dyn ControlPlaneStore,
bearer: Option<String>,
session: Option<String>,
) -> Result<Option<RequestActor>> {
if let Some(token) = bearer {
let token_hash = token_hash(&token);
if let Some(api_token) = store.resolve_api_token(&token_hash)? {
if api_token
@@ -109,7 +127,7 @@ pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
}
}
if let Some(session_token) = parse_cookie(headers, cookie_name) {
if let Some(session_token) = session {
if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? {
if is_expired(&session.expires_at) {
return Ok(None);
@@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error {
#[cfg(test)]
mod tests {
use super::*;
use crate::store::{
AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord,
};
use std::sync::Arc;
fn seeded_store() -> Arc<SqliteWorkspaceStore> {
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
workers: std::num::NonZeroUsize::new(1).unwrap(),
queued: std::num::NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let store = Arc::new(
SqliteWorkspaceStore::in_memory_for_test()
.unwrap()
.with_db_executor(executor),
);
store
.upsert_account(&AccountRecord {
account_id: "account-auth".into(),
kind: "user".into(),
handle: "auth".into(),
display_name: "Auth".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.upsert_user(&UserRecord {
user_id: "user-auth".into(),
account_id: "account-auth".into(),
handle: "auth".into(),
display_name: "Auth".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.create_browser_session(&BrowserSessionRecord {
session_id: "session-auth".into(),
token_hash: token_hash("session-secret"),
user_id: "user-auth".into(),
created_at: "1".into(),
expires_at: "9999-01-01T00:00:00Z".into(),
revoked_at: None,
})
.unwrap();
store
}
#[tokio::test]
async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() {
let store = seeded_store();
let hash = token_hash("api-secret");
store
.create_api_token(&ApiTokenRecord {
token_id: "token-auth".into(),
token_hash: hash.clone(),
user_id: "user-auth".into(),
label: "Auth".into(),
created_at: "1".into(),
expires_at: None,
revoked_at: None,
last_used_at: None,
})
.unwrap();
let mut headers = HeaderMap::new();
headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap());
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
let actor = resolve_request_actor(store.clone(), &headers, "session")
.await
.unwrap()
.unwrap();
assert_eq!(actor.user_id, "user-auth");
assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken);
assert!(
store
.resolve_api_token(&hash)
.unwrap()
.unwrap()
.last_used_at
.is_some()
);
}
#[tokio::test]
async fn expired_api_token_does_not_fall_back_to_valid_browser_session() {
let store = seeded_store();
let hash = token_hash("expired-secret");
store
.create_api_token(&ApiTokenRecord {
token_id: "token-auth".into(),
token_hash: hash.clone(),
user_id: "user-auth".into(),
label: "Auth".into(),
created_at: "1".into(),
expires_at: Some("2000-01-01T00:00:00Z".into()),
revoked_at: None,
last_used_at: None,
})
.unwrap();
let mut headers = HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
"Bearer expired-secret".parse().unwrap(),
);
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
assert!(
resolve_request_actor(store.clone(), &headers, "session")
.await
.unwrap()
.is_none()
);
assert!(
store
.resolve_api_token(&hash)
.unwrap()
.unwrap()
.last_used_at
.is_none()
);
}
#[tokio::test]
async fn unknown_bearer_can_fall_back_to_valid_browser_session() {
let store = seeded_store();
let mut headers = HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
"Bearer unknown-secret".parse().unwrap(),
);
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
let actor = resolve_request_actor(store, &headers, "session")
.await
.unwrap()
.unwrap();
assert_eq!(actor.user_id, "user-auth");
assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession);
}
#[tokio::test]
async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() {
let store = seeded_store();
let executor = store.db_executor();
let (started, started_rx) = tokio::sync::oneshot::channel();
let (release, release_rx) = std::sync::mpsc::channel();
let blocker_executor = executor.clone();
let blocker = tokio::spawn(async move {
blocker_executor
.run(move || -> Result<()> {
let _ = started.send(());
release_rx.recv().unwrap();
Ok(())
})
.await
});
tokio::time::timeout(std::time::Duration::from_secs(1), started_rx)
.await
.unwrap()
.unwrap();
let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) }));
assert!(futures::poll!(queued.as_mut()).is_pending());
let mut headers = HeaderMap::new();
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
let result = tokio::time::timeout(
std::time::Duration::from_secs(1),
resolve_request_actor(store, &headers, "session"),
)
.await;
release.send(()).unwrap();
blocker.await.unwrap().unwrap();
queued.await.unwrap();
assert!(matches!(
result.expect("capacity rejection must not wait for queued SQL"),
Err(Error::DbCapacityExceeded)
));
}
#[test]
fn user_code_uses_the_uuid_random_tail() {
+11 -4
View File
@@ -238,6 +238,13 @@ impl SqliteWorkspaceAuthority {
})
}
// Contention fixtures must hold the candidate query's actual connection,
// not the API Store used only by some enrichment paths.
#[cfg(test)]
pub(crate) fn candidate_query_store_for_test(&self) -> SqliteWorkspaceStore {
self.store.clone()
}
/// Assignment self-report enrichment is a separate projection so retained
/// responsibility and existing Ticket constructors stay distinct from status.
pub fn ticket_assignment_tracking(
@@ -3349,7 +3356,7 @@ mod tests {
async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("workspace.db");
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "w".into(),
@@ -3424,7 +3431,7 @@ mod tests {
async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("workspace.db");
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "w".into(),
@@ -3557,7 +3564,7 @@ mod tests {
write_ticket(dir.path(), "00000000001J5", "Second ticket", "queued");
write_ticket(dir.path(), "00000000001J6", "Third ticket", "planning");
let db_path = dir.path().join("workspace.db");
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace-test".to_string(),
@@ -3989,7 +3996,7 @@ VALUES ('workspace-test', 'ticket', 4);
async fn objective_mutations_write_sqlite_records_and_audit_events() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("workspace.db");
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace-test".to_string(),
+6 -6
View File
@@ -795,7 +795,7 @@ mod tests {
}
async fn open_store() -> SqliteWorkspaceStore {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
store.upsert_workspace(&workspace()).await.unwrap();
store
}
@@ -1454,7 +1454,7 @@ mod tests {
#[tokio::test]
async fn invalid_candidate_is_never_persisted() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
store.upsert_workspace(&workspace()).await.unwrap();
let current = store.load_workspace_config("w-config").unwrap().unwrap();
let main = current.snapshot.get(&path(MAIN_CONFIG_ENTRYPOINT)).unwrap();
@@ -1478,7 +1478,7 @@ mod tests {
#[tokio::test]
async fn valid_candidate_commits_snapshot_content_and_provenance_atomically() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
store.upsert_workspace(&workspace()).await.unwrap();
let current = store.load_workspace_config("w-config").unwrap().unwrap();
let committed = store
@@ -1496,7 +1496,7 @@ mod tests {
#[tokio::test]
async fn stale_cas_cannot_overwrite_newer_tree() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
store.upsert_workspace(&workspace()).await.unwrap();
let current = store.load_workspace_config("w-config").unwrap().unwrap();
let request = commit_request(&current, vec![update_main(&current, "{ answer = 42; }")]);
@@ -1514,7 +1514,7 @@ mod tests {
#[tokio::test]
async fn committed_content_remains_retrievable_after_later_commit() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
store.upsert_workspace(&workspace()).await.unwrap();
let current = store.load_workspace_config("w-config").unwrap().unwrap();
let first = store
@@ -1772,7 +1772,7 @@ mod tests {
#[test]
fn migration_creates_config_authority_tables() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
store
.with_conn(|conn| {
for table in [
+957
View File
@@ -0,0 +1,957 @@
//! Bounded execution of whole synchronous DB operations, independent of Tokio.
//!
//! Accepted jobs belong to the workers, not their async waiters. Cancelling a
//! waiter does not cancel or retry SQL, release capacity, or release its lease.
use std::{
collections::BTreeSet,
future::Future,
num::NonZeroUsize,
panic::{AssertUnwindSafe, catch_unwind},
sync::{Arc, Mutex, OnceLock, mpsc},
};
use tokio::sync::{Notify, oneshot};
use crate::{Error, Result};
/// The default is one FIFO worker and at most 32 waiting operations.
#[derive(Clone, Copy, Debug)]
pub struct DbExecutorLimits {
pub workers: NonZeroUsize,
pub queued: NonZeroUsize,
}
impl Default for DbExecutorLimits {
fn default() -> Self {
Self {
workers: NonZeroUsize::new(1).unwrap(),
queued: NonZeroUsize::new(32).unwrap(),
}
}
}
/// A request can retain its lifecycle authority until actual SQL finishes.
/// Retention happens at submission, while the caller's task-local scope exists.
pub(crate) trait DbOperationLifetime: Send + Sync {
fn retain(&self) -> Box<dyn Send>;
}
tokio::task_local! {
static OPERATION_LIFETIME: Arc<dyn DbOperationLifetime>;
}
pub(crate) async fn with_operation_lifetime<F: Future>(
lifetime: Arc<dyn DbOperationLifetime>,
future: F,
) -> F::Output {
OPERATION_LIFETIME.scope(lifetime, future).await
}
#[derive(Default)]
struct Accepted {
last_job_id: u64,
unfinished: BTreeSet<u64>,
}
#[derive(Default)]
struct Tracking {
accepted: Mutex<Accepted>,
finished: Notify,
}
struct Completion {
tracking: Arc<Tracking>,
id: u64,
}
impl Drop for Completion {
fn drop(&mut self) {
self.tracking
.accepted
.lock()
.unwrap()
.unfinished
.remove(&self.id);
self.tracking.finished.notify_waiters();
}
}
struct Job {
operation: Box<dyn FnOnce() + Send>,
// Field drop order releases authority before completion becomes observable.
_lifetime: Option<Box<dyn Send>>,
lifetime_source: Option<Arc<dyn DbOperationLifetime>>,
_completion: Completion,
runtime: Option<tokio::runtime::Handle>,
diagnostics: tracing::Dispatch,
span: tracing::Span,
}
fn report_unknown(accepted_job_id: u64, phase: &'static str) {
tracing::error!(
code = "db_operation_outcome_unknown",
accepted_job_id,
phase,
error = %Error::DbOperationOutcomeUnknown,
"DB job panicked; outcome unknown, operation will not be retried"
);
}
/// Clones share the queue and actual-job budget. Workers retain only the
/// receiver, so dropping the last sender closes the queue; accepted jobs finish
/// before the dedicated threads exit. No thread is created per store open.
#[derive(Clone)]
pub struct DbExecutor {
sender: Option<mpsc::SyncSender<Job>>,
tracking: Arc<Tracking>,
max_jobs: usize,
}
impl DbExecutor {
pub fn new(limits: DbExecutorLimits) -> Result<Self> {
let max_jobs = limits
.workers
.get()
.checked_add(limits.queued.get())
.ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?;
let (sender, receiver) = mpsc::sync_channel::<Job>(limits.queued.get());
let receiver = Arc::new(Mutex::new(receiver));
for index in 0..limits.workers.get() {
let receiver = receiver.clone();
std::thread::Builder::new()
.name(format!("workspace-db-{index}"))
.spawn(move || {
loop {
let next = receiver.lock().unwrap().recv();
let Ok(job) = next else { break };
let Job {
operation,
_lifetime: lifetime,
lifetime_source,
_completion: completion,
runtime,
diagnostics,
span,
} = job;
// Keep diagnostics with their submitting subscriber as
// well as the runtime; no process-global override.
let _diagnostics = tracing::dispatcher::set_default(&diagnostics);
let _span = span.enter();
// Preserve the submitting runtime for synchronous Host
// post-commit hooks, including reply/lease destructors.
// enter() supplies context only: never drive a future
// or await/nest an executor submission in a transaction.
let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter);
// A cancelled waiter's discarded result may itself have
// a destructor. Keep both the lease and actual-job slot
// until that cleanup finishes, even if it panics.
let execute = || {
if catch_unwind(AssertUnwindSafe(operation)).is_err() {
report_unknown(completion.id, "reply_cleanup");
}
if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() {
report_unknown(completion.id, "lifetime_cleanup");
}
};
// Synchronous post-commit continuations can admit another
// bounded job. Transfer the original lifecycle source as
// well, so its authority survives the parent job/waiter.
// This does not inject context into detached Tokio tasks.
if let Some(source) = lifetime_source {
OPERATION_LIFETIME.sync_scope(source, execute);
} else {
execute();
}
drop(completion);
}
})
.map_err(|_| Error::DbExecutorUnavailable)?;
}
Ok(Self {
sender: Some(sender),
tracking: Arc::default(),
max_jobs,
})
}
pub fn shared() -> Self {
static SHARED: OnceLock<DbExecutor> = OnceLock::new();
SHARED
.get_or_init(|| {
Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self {
sender: None,
tracking: Arc::default(),
max_jobs: 0,
})
})
.clone()
}
/// Submit without waiting for queue capacity. The closure must contain the
/// entire synchronous operation/transaction, with no nested async runtime.
/// The submitting Tokio handle, when present, is entered for synchronous
/// post-commit hooks and worker cleanup; it is never used to block_on SQL.
pub async fn run<T, E>(
&self,
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
) -> std::result::Result<T, E>
where
T: Send + 'static,
E: From<Error> + Send + 'static,
{
self.submit(operation)?
.await
.unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into()))
}
/// Admit a fire-and-forget operation immediately, without spawning a Tokio
/// task or waiting for capacity. Err means the operation was not accepted;
/// Ok means the worker owns it through execution and cleanup, even after
/// the submitting task is cancelled. A panic emits a structured unknown-
/// outcome diagnostic, never an automatic retry. drain() tracks these jobs.
pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> {
self.submit(move || {
operation();
Ok::<_, Error>(())
})
.map(drop)
}
/// Shared synchronous admission: neither dropped receivers nor cancelled
/// callers remove accepted work from the queue or the actual-job budget.
fn submit<T, E>(
&self,
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
) -> std::result::Result<oneshot::Receiver<std::result::Result<T, E>>, E>
where
T: Send + 'static,
E: From<Error> + Send + 'static,
{
let sender = self
.sender
.as_ref()
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
let runtime = tokio::runtime::Handle::try_current().ok();
let diagnostics = tracing::dispatcher::get_default(Clone::clone);
let span = tracing::Span::current();
let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok();
let lifetime = lifetime_source.as_ref().map(|source| source.retain());
let (reply, result) = oneshot::channel();
// Serialize admission and acceptance snapshots. Worker completion cannot
// race acceptance, and cancelled waiters never alter this actual-job cap.
{
let mut accepted = self.tracking.accepted.lock().unwrap();
if accepted.unfinished.len() >= self.max_jobs {
return Err(Error::DbCapacityExceeded.into());
}
let id = accepted
.last_job_id
.checked_add(1)
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
// Never reuse the identity of a rejected job: its destructor may run
// after another submission has acquired the admission lock.
accepted.last_job_id = id;
let job = Job {
operation: Box::new(move || {
let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| {
report_unknown(id, "operation");
Err(Error::DbOperationOutcomeUnknown.into())
});
let _ = reply.send(outcome);
}),
_lifetime: lifetime,
lifetime_source,
runtime,
diagnostics,
span,
_completion: Completion {
tracking: self.tracking.clone(),
id,
},
};
match sender.try_send(job) {
Ok(()) => {
accepted.unfinished.insert(id);
}
Err(error) => {
// The rejected Job's completion also locks tracking; drop it only
// after releasing the admission lock.
drop(accepted);
return Err(match error {
mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded,
mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable,
}
.into());
}
}
}
Ok(result)
}
/// Wait for every job accepted before this barrier is first polled (not
/// later submissions). This bypasses queue capacity and survives cancelled
/// job waiters. It does not close the queue or stop new submissions.
pub async fn drain(&self) -> Result<()> {
if self.sender.is_none() {
return Err(Error::DbExecutorUnavailable);
}
let through = self.tracking.accepted.lock().unwrap().last_job_id;
loop {
let notified = self.tracking.finished.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self
.tracking
.accepted
.lock()
.unwrap()
.unfinished
.first()
.is_none_or(|id| *id > through)
{
return Ok(());
}
notified.await;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use rusqlite::{Connection, TransactionBehavior, ffi};
use std::{ffi::c_void, time::Duration};
fn executor(queued: usize) -> DbExecutor {
DbExecutor::new(DbExecutorLimits {
workers: NonZeroUsize::new(1).unwrap(),
queued: NonZeroUsize::new(queued).unwrap(),
})
.unwrap()
}
fn database() -> (tempfile::TempDir, Connection, Connection) {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("jobs.sqlite");
let owner = Connection::open(&path).unwrap();
owner
.execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)")
.unwrap();
let worker = Connection::open(&path).unwrap();
(dir, owner, worker)
}
struct BusyGate {
waiting: Option<oneshot::Sender<()>>,
release: mpsc::Receiver<()>,
}
// A fixture-owned callback proves SQLite actually reached its lock wait.
// Unlike busy_handler's fn pointer API, the SQLite API carries a per-call
// context, so tests need neither process-global state nor guessed sleeps.
struct BusyRegistration {
db: *mut ffi::sqlite3,
_gate: Box<BusyGate>,
}
impl BusyRegistration {
fn install(conn: &Connection, gate: BusyGate) -> Self {
unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 {
// The Box remains registered only while this connection and
// BusyRegistration are alive, on this same dedicated worker.
let gate = unsafe { &mut *context.cast::<BusyGate>() };
if let Some(waiting) = gate.waiting.take() {
let _ = waiting.send(());
if gate.release.recv().is_ok() {
return 1;
}
}
0
}
let mut gate = Box::new(gate);
// SAFETY: conn owns the handle; registration is dropped before conn.
let db = unsafe { conn.handle() };
let result = unsafe {
ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast())
};
assert_eq!(result, ffi::SQLITE_OK);
Self { db, _gate: gate }
}
}
impl Drop for BusyRegistration {
fn drop(&mut self) {
// SAFETY: unregister before releasing the callback's boxed context.
unsafe {
ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut());
}
}
}
async fn observed<T>(future: impl Future<Output = T>) -> T {
tokio::time::timeout(Duration::from_secs(5), future)
.await
.expect("DB job did not reach the expected observable state")
}
#[tokio::test(flavor = "current_thread")]
async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() {
let (_dir, mut owner, mut worker) = database();
let lock = owner
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let executor = executor(1);
let (waiting, blocked) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut write = Box::pin(executor.run(move || -> Result<()> {
let _busy = BusyRegistration::install(
&worker,
BusyGate {
waiting: Some(waiting),
release: permit,
},
);
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
Ok(())
}));
assert!(futures::poll!(&mut write).is_pending());
observed(blocked).await.unwrap();
let control = tokio::spawn(async { 42 });
assert_eq!(observed(control).await.unwrap(), 42);
// The timer itself is the contract under test, not synchronization.
observed(tokio::time::sleep(Duration::from_millis(1))).await;
assert!(futures::poll!(&mut write).is_pending());
lock.rollback().unwrap();
release.send(()).unwrap();
observed(write).await.unwrap();
executor.drain().await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
}
struct Lease(Arc<std::sync::atomic::AtomicUsize>);
struct Retained(Arc<std::sync::atomic::AtomicUsize>);
impl DbOperationLifetime for Lease {
fn retain(&self) -> Box<dyn Send> {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Box::new(Retained(self.0.clone()))
}
}
impl Drop for Retained {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() {
let (_dir, mut owner, mut worker) = database();
let lock = owner
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let executor = executor(1);
let (waiting, blocked) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut running = Box::pin(executor.run(move || -> Result<()> {
let _busy = BusyRegistration::install(
&worker,
BusyGate {
waiting: Some(waiting),
release: permit,
},
);
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
Ok(())
}));
assert!(futures::poll!(&mut running).is_pending());
observed(blocked).await.unwrap();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap();
let mut queued = Box::pin(with_operation_lifetime(
lease,
executor.run(move || -> Result<()> {
let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
// FIFO is observable in durable SQL, not inferred from waiter order.
let first: i64 =
tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| {
row.get(0)
})?;
assert_eq!(first, 1);
tx.execute("INSERT INTO writes VALUES (2)", [])?;
tx.commit()?;
Ok(())
}),
));
assert!(futures::poll!(&mut queued).is_pending());
drop(queued);
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
assert!(matches!(
executor.run(|| Ok::<_, Error>(())).await,
Err(Error::DbCapacityExceeded)
));
let mut drained = Box::pin(executor.drain());
assert!(futures::poll!(&mut drained).is_pending());
lock.rollback().unwrap();
release.send(()).unwrap();
observed(running).await.unwrap();
observed(drained).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
2
);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() {
let (dir, owner, mut parent_conn) = database();
let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap();
let parent = executor(1);
let child = executor(1);
let child_owner = child.clone();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let source: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let (entered, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let (child_entered, child_running) = oneshot::channel();
let (finish_child, child_permit) = mpsc::channel();
let mut operation = Box::pin(with_operation_lifetime(
source,
parent.run(move || -> Result<()> {
let _ = entered.send(());
let _ = permit.recv();
let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
child_owner.dispatch(move || {
let tx = child_conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let committed: i64 = tx
.query_row("SELECT count(*) FROM writes", [], |row| row.get(0))
.unwrap();
assert_eq!(committed, 1, "child must see its parent's committed SQL");
let _ = child_entered.send(());
let _ = child_permit.recv();
tx.execute("INSERT INTO writes VALUES (2)", []).unwrap();
tx.commit().unwrap();
})?;
Ok(())
}),
));
assert!(futures::poll!(&mut operation).is_pending());
observed(running).await.unwrap();
drop(operation);
release.send(()).unwrap();
observed(child_running).await.unwrap();
observed(parent.drain()).await.unwrap();
assert_eq!(
count.load(std::sync::atomic::Ordering::SeqCst),
1,
"accepted continuation lost authority when its parent finished"
);
let mut drained = Box::pin(child.drain());
assert!(futures::poll!(&mut drained).is_pending());
finish_child.send(()).unwrap();
observed(drained).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
2
);
}
#[tokio::test]
async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() {
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let error = executor
.run(move || -> Result<()> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.execute("INSERT INTO writes VALUES (NULL)", [])?;
tx.commit()?;
Ok(())
})
.await
.unwrap_err();
assert!(matches!(error, Error::Sqlite(_)));
executor.drain().await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
0
);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[tokio::test]
async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() {
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let error = executor
.run(move || -> Result<()> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
panic!("lost outcome after commit");
})
.await
.unwrap_err();
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
executor.drain().await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42);
}
#[tokio::test]
async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() {
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let error = with_operation_lifetime(
lease,
executor.run(move || -> Result<()> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
panic!("lost outcome before commit");
}),
)
.await
.unwrap_err();
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
executor.drain().await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
0
);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
struct DiscardedReplyHook {
expected_runtime: tokio::runtime::Id,
lease_count: Arc<std::sync::atomic::AtomicUsize>,
dropped: Option<oneshot::Sender<(bool, usize)>>,
}
impl Drop for DiscardedReplyHook {
fn drop(&mut self) {
let runtime =
tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context");
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
let same_runtime = runtime.id() == self.expected_runtime;
let dropped = self.dropped.take().unwrap();
tokio::spawn(async move {
let _ = dropped.send((same_runtime, retained));
});
}
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop()
{
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let (started, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
let _ = started.send(());
let _ = permit.recv();
Ok(())
}));
assert!(futures::poll!(&mut blocker).is_pending());
observed(running).await.unwrap();
let expected_runtime = tokio::runtime::Handle::current().id();
let (hook, published) = oneshot::channel();
let (finish, finishing) = mpsc::channel();
let (dropped, reply_dropped) = oneshot::channel();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let operation_count = count.clone();
let mut queued = Box::pin(with_operation_lifetime(
lease,
executor.run(move || -> Result<_> {
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
// Model synchronous Host hooks: publish only after commit, checking
// try_current and spawning without driving any future on this worker.
let runtime = tokio::runtime::Handle::try_current()
.expect("post-commit hook lost runtime context");
assert_eq!(runtime.id(), expected_runtime);
tokio::spawn(async move {
let _ = hook.send(());
});
let _ = finishing.recv();
Ok(DiscardedReplyHook {
expected_runtime,
lease_count: operation_count,
dropped: Some(dropped),
})
}),
));
assert!(futures::poll!(&mut queued).is_pending());
drop(queued);
release.send(()).unwrap();
observed(blocker).await.unwrap();
observed(published).await.unwrap();
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
let mut drain = Box::pin(executor.drain());
assert!(futures::poll!(&mut drain).is_pending());
finish.send(()).unwrap();
assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1));
observed(drain).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() {
let executor = DbExecutor::new(DbExecutorLimits {
workers: NonZeroUsize::new(2).unwrap(),
queued: NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let mut releases = Vec::new();
// Observe each worker executing before submitting the next operation,
// so queue occupancy and overload are deterministic, not timing guesses.
for _ in 0..2 {
let (started, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut operation = Box::pin(executor.run(move || -> Result<()> {
let _ = started.send(());
let _ = permit.recv();
Ok(())
}));
assert!(futures::poll!(&mut operation).is_pending());
observed(running).await.unwrap();
drop(operation);
releases.push(release);
}
let (finished, mut queued_finished) = oneshot::channel();
let mut queued = Box::pin(executor.run(move || -> Result<()> {
let _ = finished.send(());
Ok(())
}));
assert!(futures::poll!(&mut queued).is_pending());
drop(queued);
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
for _ in 0..8 {
let rejected_runs = rejected_runs.clone();
assert!(matches!(
executor
.run(move || {
rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok::<_, Error>(())
})
.await,
Err(Error::DbCapacityExceeded)
));
}
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
assert!(matches!(
queued_finished.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
let mut drain = Box::pin(executor.drain());
assert!(futures::poll!(&mut drain).is_pending());
for release in releases {
release.send(()).unwrap();
}
observed(drain).await.unwrap();
observed(queued_finished).await.unwrap();
executor.run(|| Ok::<_, Error>(())).await.unwrap();
}
#[derive(Default)]
struct UnknownDiagnostic {
code: Option<String>,
phase: Option<String>,
accepted_job_id: Option<u64>,
}
impl tracing::field::Visit for UnknownDiagnostic {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
match field.name() {
"code" => self.code = Some(value.to_owned()),
"phase" => self.phase = Some(value.to_owned()),
_ => {}
}
}
fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
if field.name() == "accepted_job_id" {
self.accepted_job_id = Some(value);
}
}
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
}
#[derive(Clone, Default)]
struct DiagnosticCapture(Arc<Mutex<Vec<UnknownDiagnostic>>>);
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for DiagnosticCapture {
fn on_event(
&self,
event: &tracing::Event<'_>,
_: tracing_subscriber::layer::Context<'_, S>,
) {
let mut diagnostic = UnknownDiagnostic::default();
event.record(&mut diagnostic);
self.0.lock().unwrap().push(diagnostic);
}
}
struct DispatchCleanup {
expected_runtime: tokio::runtime::Id,
lease_count: Arc<std::sync::atomic::AtomicUsize>,
started: Option<oneshot::Sender<(bool, usize)>>,
release: mpsc::Receiver<()>,
}
impl Drop for DispatchCleanup {
fn drop(&mut self) {
// No panicking assertions during unwind: report observations to the
// test, and gate cleanup so drain/lifetime behavior is observable.
let same_runtime = tokio::runtime::Handle::try_current()
.is_ok_and(|runtime| runtime.id() == self.expected_runtime);
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
if let Some(started) = self.started.take() {
let _ = started.send((same_runtime, retained));
}
let _ = self.release.recv();
}
}
#[tokio::test(flavor = "current_thread")]
async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry()
{
use tracing_subscriber::layer::SubscriberExt;
let (_dir, owner, mut worker) = database();
let executor = executor(1);
let (started, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
let _ = started.send(());
let _ = permit.recv();
Ok(())
}));
assert!(futures::poll!(&mut blocker).is_pending());
observed(running).await.unwrap();
drop(blocker); // Cancellation does not free its running slot.
let expected_runtime = tokio::runtime::Handle::current().id();
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
let operation_count = count.clone();
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let operation_attempts = attempts.clone();
let (cleanup_started, cleaning_up) = oneshot::channel();
let (finish, finishing) = mpsc::channel();
// Model multiple tracing contexts without setting a global subscriber.
// This also prevents tracing's single-subscriber callsite optimization
// from caching an unsubscribed parallel test's Interest::never here.
let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry());
let captured = DiagnosticCapture::default();
let diagnostics =
tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone()));
with_operation_lifetime(lease, async {
tracing::dispatcher::with_default(&diagnostics, || {
executor.dispatch(move || {
let _cleanup = DispatchCleanup {
expected_runtime,
lease_count: operation_count,
started: Some(cleanup_started),
release: finishing,
};
operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let tx = worker
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
tx.execute("INSERT INTO writes VALUES (1)", []).unwrap();
tx.commit().unwrap();
panic!("dispatched post-commit outcome lost");
})
})
})
.await
.unwrap(); // Dispatch is admitted synchronously; no waiter survives.
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let rejected_operation = rejected_runs.clone();
assert!(matches!(
executor.dispatch(move || {
rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}),
Err(Error::DbCapacityExceeded)
));
release.send(()).unwrap();
assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1));
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |row| row
.get::<_, i64>(0))
.unwrap(),
1
);
let mut drain = Box::pin(executor.drain());
assert!(futures::poll!(&mut drain).is_pending());
finish.send(()).unwrap();
observed(drain).await.unwrap();
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
{
let events = captured.0.lock().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(
events[0].code.as_deref(),
Some("db_operation_outcome_unknown")
);
assert_eq!(events[0].phase.as_deref(), Some("operation"));
assert!(events[0].accepted_job_id.is_some());
}
let (completed, completion) = oneshot::channel();
executor
.dispatch(move || {
let _ = completed.send(());
})
.unwrap();
observed(executor.drain()).await.unwrap();
observed(completion).await.unwrap();
}
}
@@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
migrate_content_state_v85_to_v86(&conn).unwrap();
drop(conn);
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
};
let job = store
@@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
migrate_content_state_v85_to_v86(&conn).unwrap();
drop(conn);
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
};
let job = store
@@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
};
let migrated = store
@@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
});
let archived = store
@@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
});
let service =
@@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() {
// The service consumes current receipt columns; unrelated schemas are not this fixture.
rename_saved_secret_receipt_counters(&conn).unwrap();
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
});
let service =
@@ -533,6 +533,7 @@ mod existing_workspace_provisioning {
// This fixture contains only the schema-87 cutover authorities, not the
// unrelated full Server schema. Exercise the real service/store methods.
Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(conn)),
})
}
@@ -644,6 +645,7 @@ mod existing_workspace_provisioning {
drop(service);
drop(store);
let store = Arc::new(SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())),
});
let service = WorkspaceSigningIdentityService::new(
@@ -816,7 +818,7 @@ mod current_authority {
use super::*;
fn binding_store() -> (SqliteWorkspaceStore, WorkspaceRuntimeBinding) {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
let runtime_key =
worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-key").unwrap();
store.with_conn(|conn| {
+122 -15
View File
@@ -23,7 +23,10 @@ use std::{
};
use workdir::{
Workdir, WorkdirError, WorkdirSessionHandle,
http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization},
http::{
OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization,
WorkdirHttpAuthorizationFuture,
},
};
use worker_runtime::RuntimeWorkspaceScope;
use worker_runtime::catalog::{
@@ -3477,6 +3480,38 @@ impl WorkspaceRuntimeAuthorization {
}
}
/// Offload the complete live-binding check and request signing operation.
/// Claims are minted inside the bounded job, not cached before admission.
pub(crate) async fn issue_async(
&self,
method: String,
path_and_query: String,
operation: String,
worker_id: Option<String>,
body: Vec<u8>,
) -> Result<String, RuntimeDiagnostic> {
let executor = self.store.db_executor();
let authorization = self.clone();
executor
.run(move || {
Ok::<_, Error>(authorization.issue(
&method,
&path_and_query,
&operation,
worker_id.as_deref(),
&body,
))
})
.await
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
HostDiagnosticSeverity::Error,
error.to_string(),
)
})?
}
pub(crate) fn issue(
&self,
method: &str,
@@ -3649,26 +3684,25 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization {
impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization {
fn bearer_token(
&self,
method: &str,
path_and_query: &str,
body: &[u8],
) -> Result<String, WorkdirError> {
if let Some(authorization) = &self.workspace_authorization {
return authorization
.issue(
method,
path_and_query,
workspace_runtime_operation(method, path_and_query),
None,
body,
)
method: String,
path_and_query: String,
body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture {
let authorization = self.clone();
Box::pin(async move {
if let Some(authorizer) = &authorization.workspace_authorization {
let operation = workspace_runtime_operation(&method, &path_and_query).to_string();
return authorizer
.issue_async(method, path_and_query, operation, None, body)
.await
.map_err(|error| WorkdirError::Unavailable(error.message));
}
self.fallback_bearer_token.clone().ok_or_else(|| {
authorization.fallback_bearer_token.ok_or_else(|| {
WorkdirError::Unavailable(
"remote Runtime does not have bearer authorization configured".to_string(),
)
})
})
}
}
@@ -6348,6 +6382,79 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::thread;
#[tokio::test]
async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() {
use crate::db_executor::{DbExecutor, DbExecutorLimits};
use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding};
use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore;
let executor = DbExecutor::new(DbExecutorLimits {
workers: std::num::NonZeroUsize::new(1).unwrap(),
queued: std::num::NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let store = Arc::new(
SqliteWorkspaceStore::in_memory_for_test()
.unwrap()
.with_db_executor(executor.clone()),
);
let service = WorkspaceSigningIdentityService::new(
store.clone(),
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
);
let authorization = RemoteWorkdirAuthorization {
workspace_authorization: Some(WorkspaceRuntimeAuthorization::new(
store,
service,
"https://backend.test",
WorkspaceRuntimeBinding {
workspace_id: "workspace".into(),
runtime_id: "runtime".into(),
display_name: "Runtime".into(),
base_url: "https://runtime.test".into(),
public_key_fingerprint: "unused".into(),
public_key: "unused".into(),
binding_id: "binding".into(),
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
created_at: "1".into(),
updated_at: "1".into(),
revoked_at: None,
},
)),
fallback_bearer_token: Some("must-not-be-used".into()),
};
// Hold the running slot and fill the only queue slot. The callback must
// reject admission, not run inline or silently switch authentication.
let (release, wait) = std::sync::mpsc::channel::<()>();
let (started, running) = tokio::sync::oneshot::channel();
executor
.dispatch(move || {
let _ = started.send(());
let _ = wait.recv_timeout(Duration::from_secs(5));
})
.unwrap();
tokio::time::timeout(Duration::from_secs(5), running)
.await
.unwrap()
.unwrap();
executor.dispatch(|| {}).unwrap();
let error = authorization
.bearer_token(
"POST".into(),
"/v1/workdir-sessions/s/operations".into(),
b"{}".to_vec(),
)
.await
.unwrap_err();
assert!(matches!(
error,
WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string()
));
drop(release);
executor.drain().await.unwrap();
}
#[test]
fn ticket_worker_spawn_intent_uses_worker_wire_role() {
let intent = WorkerSpawnIntent::TicketRole {
@@ -32,7 +32,8 @@ struct Fixture {
impl Fixture {
async fn new() -> Self {
let temp = tempfile::tempdir().unwrap();
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap());
let store =
Arc::new(SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap());
store
.upsert_account(&AccountRecord {
account_id: "owner".into(),
@@ -45,7 +45,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() {
let artifacts = evidence_rows(&conn, "artifacts");
let receipts = evidence_rows(&conn, "repository_secret_legacy_receipts");
drop(conn);
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
upgraded
.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
@@ -73,7 +73,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() {
Ok(())
})
.unwrap();
for store in [upgraded, SqliteWorkspaceStore::in_memory().unwrap()] {
for store in [upgraded, SqliteWorkspaceStore::in_memory_for_test().unwrap()] {
store.with_conn(|conn| {
let mut stmt = conn.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")?;
let objects = stmt.query_map([],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
+7
View File
@@ -10,6 +10,7 @@ pub mod backend_job;
pub mod companion;
pub mod config;
pub mod config_source;
pub mod db_executor;
pub mod feature_storage;
pub mod hosts;
pub mod memory_backend;
@@ -159,6 +160,12 @@ pub enum Error {
WorkspaceSigningIdentity { code: String, message: String },
#[error("Workspace Drive error: {0}")]
Drive(#[from] workspace_drive::Error),
#[error("DB execution capacity exceeded")]
DbCapacityExceeded,
#[error("DB executor unavailable")]
DbExecutorUnavailable,
#[error("DB operation outcome unknown; do not automatically retry")]
DbOperationOutcomeUnknown,
#[error("store error: {0}")]
Store(String),
}
@@ -217,7 +217,7 @@ mod tests {
async fn authority() -> (TempDir, SqliteWorkspaceAuthority) {
let temp = TempDir::new().unwrap();
let db_path = temp.path().join("workspace.db");
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace-test".to_string(),
@@ -32,7 +32,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() {
drop(conn);
for attempt in 0..3 {
let error = match attempt {
0 => SqliteWorkspaceStore::open(&path).err().unwrap(),
0 => SqliteWorkspaceStore::open_for_test(&path).err().unwrap(),
1 => SqliteWorkspaceStore::migration_plan(&path).unwrap_err(),
_ => SqliteWorkspaceStore::migrate_database(&path).unwrap_err(),
};
@@ -62,7 +62,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() {
)
.unwrap();
drop(conn);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
assert_eq!(conn.query_row("SELECT input_fingerprint FROM worker_removal_operations WHERE operation_id='removal'",[],|r|r.get::<_,String>(0))?,"legacy-receipt");
@@ -109,7 +109,7 @@ fn migration_dry_run_and_apply_share_the_original_repository_secret_authority()
assert!(!table_exists(&conn, "server_secret_objects").unwrap());
drop(conn);
assert_eq!(SqliteWorkspaceStore::migrate_database(&path).unwrap(), plan);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?,LATEST_SCHEMA_VERSION);
let (nonce, mut sealed): (Vec<u8>,Vec<u8>) = conn.query_row(
@@ -2290,7 +2290,7 @@ mod tests {
) {
let dir = tempfile::tempdir().unwrap();
let database = dir.path().join("server.db");
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
let timestamp = now();
for workspace_id in ["workspace-a", "workspace-b"] {
futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord {
+1 -1
View File
@@ -1116,7 +1116,7 @@ mod tests {
WorkerId::from_legacy_u64(1)
}
fn setup() -> SqliteWorkspaceStore {
let s = SqliteWorkspaceStore::in_memory().unwrap();
let s = SqliteWorkspaceStore::in_memory_for_test().unwrap();
s.with_conn(|c| {
c.execute(
"INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) \
@@ -965,7 +965,7 @@ async fn connect_runtime(
let mut request = endpoint
.into_client_request()
.map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?;
if let Some(token) = runtime_token(config, workspace_id)? {
if let Some(token) = runtime_token(config, workspace_id).await? {
request.headers_mut().insert(
"authorization",
format!("Bearer {token}")
@@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String {
format!("{base}/v1/protocol/ws")
}
}
fn runtime_token(
async fn runtime_token(
config: &RemoteRuntimeConfig,
_workspace_id: &str,
) -> Result<Option<String>, String> {
if let Some(authorization) = config.workspace_authorization.as_ref() {
return authorization
.issue("GET", "/v1/protocol/ws", "workers:list", None, &[])
.issue_async(
"GET".to_string(),
"/v1/protocol/ws".to_string(),
"workers:list".to_string(),
None,
Vec::new(),
)
.await
.map(Some)
.map_err(|error| error.message);
}
@@ -38,8 +38,8 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("upgraded.db");
prepare_retained_schema(&path, 85);
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
let fresh = SqliteWorkspaceStore::in_memory().unwrap();
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
let mut mismatches = Vec::new();
for table in [
"workspace_tags",
File diff suppressed because it is too large Load Diff
@@ -65,12 +65,20 @@ pub(super) fn require_candidate_grant(
/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface
/// background work; the new Job regenerates it and does not infer surface success.
pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult<bool> {
let job_api = api.clone();
let subject_id = subject_id.to_owned();
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
let (worker, complete) = api
.host_executors
.host()?
.run(move || {
let api = &job_api;
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
let Some(owner) = api
.store
.current_worker_singleton_owner(api.workspace_id(), &key)?
else {
return Ok(true);
return Ok((None, true));
};
let mut summary = api
.runtime
@@ -85,7 +93,7 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
.as_ref()
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
if !stopped && !idle {
return Ok(false);
return Ok((None, false));
}
if api
.store
@@ -106,14 +114,15 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
&owner.worker,
WorkerLifecycleRequest {
reason: Some(
"Retire legacy subjektiv consolidator after foreground completion".into(),
"Retire legacy subjektiv consolidator after foreground completion"
.into(),
),
ticket_assignment: None,
},
)
.map_err(|e| e.into_error())?;
if stopped.state != InternalWorkerOperationState::Accepted {
return Ok(false);
return Ok((None, false));
}
}
if !api
@@ -123,12 +132,18 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
.state
.eq_ignore_ascii_case("stopped")
{
return Ok(false);
return Ok((None, false));
}
Ok::<_, ApiError>((Some(owner.worker), false))
})
.await?;
let Some(worker) = worker else {
return Ok(complete);
};
let removal = WorkerRemovalService::new(api)
.execute_target_removal(
api.runtime.as_ref(),
&owner.worker,
&worker,
"subjektiv consolidation Job cutover",
None,
)
@@ -141,10 +156,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
))
.into());
}
Ok(api
.store
.current_worker_singleton_owner(api.workspace_id(), &key)?
.is_none())
let store = api.store.clone();
let workspace_id = api.workspace_id().to_owned();
api.store
.db_executor()
.run(move || {
store
.current_worker_singleton_owner(&workspace_id, &key)
.map(|owner| owner.is_none())
.map_err(ApiError::from)
})
.await
}
fn bounded_candidate_batch(
@@ -806,7 +806,19 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc
.unwrap();
};
let ((status, body, events), ()) = tokio::join!(
request_with_logs(build_router(fixture.api.clone()), request),
async {
let result = request_with_logs(build_router(fixture.api.clone()), request).await;
// A pre-dispatch rejection leaves the fake provider waiting on an
// open channel. Fail with the actual response here instead of
// hanging join! until the outer suite timeout hides that cause.
assert_eq!(
result.0,
StatusCode::BAD_GATEWAY,
"early command-start response: {}",
result.1
);
result
},
provider
);
assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}");
@@ -832,3 +844,78 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc
"post-dispatch failure was retried"
);
}
#[tokio::test]
async fn saturated_host_before_command_start_returns_availability_without_a_provider_operation() {
let mut fixture = manual_worker_assignment_fixture().await;
let identity =
worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id)
.unwrap();
configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id);
let (sender, mut commands) = tokio::sync::mpsc::channel(1);
let connection = Arc::new(ExternalProviderConnection {
grant_id: "overload-grant".into(),
workdir_id: fixture.main_workdir_id.clone(),
provider_instance_id: "overload-provider".into(),
generation: 1,
expires_at: None,
capabilities: workdir::WorkdirSessionCapabilities::ALL,
admission: Arc::new(tokio::sync::Semaphore::new(1)),
shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
sender,
});
fixture.runtime.workdir_session_factory = Some(Arc::new(move |_| {
Ok(Arc::new(ExternalProviderWorkdirSession::new(
connection.clone(),
None,
)) as WorkdirSessionHandle)
}));
fixture
.api
.runtime
.register_or_replace(fixture.runtime.clone());
let host = fixture.api.host_executors.host().unwrap();
host.drain().await.unwrap();
let mut releases = Vec::new(); // dropping senders also releases jobs on panic
for _ in 0..2 {
let (release, permit) = std::sync::mpsc::channel::<()>();
releases.push(release);
let (started, accepted) = tokio::sync::oneshot::channel();
host.dispatch(move || {
let _ = started.send(());
let _ = permit.recv_timeout(std::time::Duration::from_secs(10));
})
.unwrap();
accepted.await.unwrap();
}
for _ in 0..16 {
host.dispatch(|| {}).unwrap();
}
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations");
let request = runtime_source_request(
&identity,
Some(&fixture.worker.worker_id),
"POST",
&path,
serde_json::to_vec(&json!({
"target_workdir": "checkout",
"operation": {"operation": "command_start", "request": {
"command": "not-dispatched", "timeout_secs": 1, "output_limit": 1024,
"cwd": "", "spill_dir": null, "tool_call_id": "not-dispatched"
}}
}))
.unwrap(),
);
let (status, body, _) = request_with_logs(build_router(fixture.api.clone()), request).await;
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");
assert!(body.to_string().contains("db_capacity_exceeded"), "{body}");
// The response is already final but the live fake-provider receiver remains
// Pending. An unbounded join of those two futures hid this causal rejection
// behind the original 600-second suite timeout. No process or command ran.
let provider_command = commands.recv();
tokio::pin!(provider_command);
assert!(futures::poll!(provider_command.as_mut()).is_pending());
drop(releases);
host.drain().await.unwrap();
fixture.api.store.db_executor().drain().await.unwrap();
}
@@ -54,7 +54,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
configure_runtime_request_auth(&mut api, &identity, "runtime-test");
let revoked = RuntimeIdentityMaterial::generate("revoked-runtime-secret").unwrap();
configure_runtime_request_auth(&mut api, &revoked, "revoked-runtime-secret");
SqliteWorkspaceStore::open(&api.config.database_path)
SqliteWorkspaceStore::open_for_test(&api.config.database_path)
.unwrap()
.revoke_workspace_runtime_binding(
TEST_WORKSPACE_ID,
@@ -70,7 +70,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
let body = "body-secret";
let apps = [
build_router(api.clone()),
workspace_server_router(WorkspaceServerApi::new(
workspace_server_router(WorkspaceServerApi::new_for_test(
api.config.clone(),
api.store.clone(),
)),
@@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m
&error,
)
});
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(
to_bytes(response.into_body(), usize::MAX)
.await
@@ -347,7 +347,7 @@ async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() {
let path = format!("/api/w/{TEST_WORKSPACE_ID}/repositories");
for app in [
build_router(api.clone()),
workspace_server_router(WorkspaceServerApi::new(
workspace_server_router(WorkspaceServerApi::new_for_test(
api.config.clone(),
api.store.clone(),
)),
@@ -40,7 +40,7 @@ async fn cached_drive_handle_rejects_mutation_after_deletion_reservation() {
})
.await
.unwrap();
let api = WorkspaceApi::new(config.clone(), store.clone())
let api = WorkspaceApi::new_for_test(config.clone(), store.clone())
.await
.unwrap();
let cached = api.drive().await.unwrap();
@@ -104,7 +104,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature
// Use a separate embedded-runtime namespace for the first API: this test
// reconstructs Drive authority in-process rather than spawning/restarting
// product processes (runtime observers may retain their namespace).
let first_api = WorkspaceApi::new(config.clone(), store.clone())
let first_api = WorkspaceApi::new_for_test(config.clone(), store.clone())
.await
.unwrap();
// Merely constructing a Workspace API must not create blob storage.
@@ -160,7 +160,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature
.unwrap()
.unwrap();
drop(first_api);
let recovered = WorkspaceServerApi::new(config.clone(), store.clone());
let recovered = WorkspaceServerApi::new_for_test(config.clone(), store.clone());
let api = recovered
.api_for_workspace(&config.workspace_id)
.await
@@ -39,10 +39,10 @@ impl RealDriveFixture {
let temp = tempfile::tempdir().unwrap();
let mut config = test_server_config(temp.path());
config.repositories.clear();
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
let owner = seed_test_api_token(store.as_ref(), "drive-web-owner");
let second = seed_test_api_token(store.as_ref(), "drive-web-second");
let server_api = WorkspaceServerApi::new(config, store.clone());
let server_api = WorkspaceServerApi::new_for_test(config, store.clone());
let server = LoopbackServer::start(workspace_server_router(server_api.clone())).await;
// Exercise T-717's real repositoryless Workspace bootstrap, not a
// pre-existing test Workspace with its repository merely hidden in UI.
@@ -767,6 +767,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) {
std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 })
);
assert!(execution.protocol_methods().is_empty());
// SQLite jobs run on external threads; paused Tokio time cannot observe their
// progress and may auto-advance a reconnect timeout before SQL completes.
tokio::time::resume();
assert_same_worker_reconnect_and_stop(&worker, &context).await;
}
@@ -844,7 +847,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub
}
}
#[tokio::test(start_paused = true)]
#[tokio::test]
async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress()
{
let workspace = tempfile::tempdir().unwrap();
@@ -27,7 +27,7 @@ async fn subject_session_callback_completes_while_create_and_deletion_are_waitin
)
.unwrap();
let worker_id = reserved.worker_id.to_string();
let server = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
// Reuse the fixture API, but exercise the real outer dispatcher and both auth layers.
server
.apis
@@ -144,7 +144,7 @@ impl LaunchFixture {
async fn new(remote: bool) -> Self {
let temp = tempfile::tempdir().unwrap();
let config = test_server_config(temp.path());
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap();
let backend = Arc::new(LaunchExecutionBackend::default());
let api =
WorkspaceApi::new_with_execution_backend(config, Arc::new(store), backend.clone())
@@ -151,7 +151,7 @@ impl Fixture {
let temp = tempfile::tempdir().unwrap();
let mut config = test_server_config(temp.path());
config.repositories.clear();
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
let mut api = WorkspaceApi::new_with_execution_backend(
config,
store,
@@ -830,7 +830,11 @@ async fn grant_replacement_while_admission_waits_cannot_use_another_grants_lock(
};
let admission = handle.set_pinned(&context, true);
tokio::pin!(admission);
// Poll deterministically to the held lock before replacing grant authority.
// The first poll can now be waiting for the offloaded authority read, not
// the held grant lock. Drain this fixture's actual SQL, then poll its saved
// reply to reach admission before replacing authority.
assert!(futures::poll!(admission.as_mut()).is_pending());
api.store.db_executor().drain().await.unwrap();
assert!(futures::poll!(admission.as_mut()).is_pending());
api.store
.revoke_worker_control_grant(
@@ -1329,8 +1333,9 @@ async fn restore_guard_internal_stale_conflict_retires_intent_for_later_explicit
);
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
// Retirement, unlike in-memory refresh, must survive caller journal restart.
api.config_store =
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
api.config_store = Arc::new(
SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(),
);
let fresh = api.runtime.worker(&identity).unwrap();
let next = api
.config_store
@@ -1490,7 +1495,7 @@ async fn restore_guard_internal_intent_survives_client_store_restart_without_rep
);
// Reopen the durable caller journal, discarding all client-side memory of request identity.
api.config_store =
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
Arc::new(SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap());
let pinned = api
.config_store
.pin_internal_worker_restore_intent(
@@ -1599,7 +1604,7 @@ async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_w
crate::repository_source::repository_source_fingerprint(&source);
config.repositories[0].source = source;
config.repositories[0].path = None;
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap();
let execution = Arc::new(DeterministicExecutionBackend::default());
execution.accept_restores();
let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone())
@@ -56,7 +56,7 @@ impl Fixture {
api.runtime.register_or_replace(runtime);
let now = Arc::new(AtomicI64::new(START));
let clock = now.clone();
let mut server = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
let mut server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
server.proof_clock = Arc::new(move || clock.load(Ordering::SeqCst));
server
.apis
@@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
let f = Fixture::new().await;
let mut a = f.attach("1");
let (first, reply) = f.start_read("1", &mut a).await;
let mut second = Box::pin(f.app.clone().oneshot(f.operation("1")));
// Poll through dispatch/auth until the handler waits on its resource lock.
let request = f.operation("1");
let proof = request
.headers()
.get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER)
.unwrap()
.to_str()
.unwrap();
let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof)
.unwrap()
.jti;
let mut second = Box::pin(f.app.clone().oneshot(request));
// DB auth is now genuinely asynchronous. Observe durable proof consumption,
// not a guessed number of polls, before advancing the injected proof clock.
let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
tokio::time::timeout(BOUND, async {
loop {
assert!(futures::poll!(&mut second).is_pending());
let consumed: bool = connection
.query_row(
"SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)",
[&jti],
|row| row.get(0),
)
.unwrap();
if consumed {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("second request never authenticated before resource wait");
assert!(
a.try_recv().is_err(),
"same session read escaped resource exclusion"
@@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
assert_json(first.await.unwrap(), StatusCode::OK).await;
assert!(futures::poll!(&mut second).is_pending());
let Some(ExternalProviderCommand::Operation { response, .. }) =
tokio::time::timeout(BOUND, a.recv()).await.unwrap()
tokio::time::timeout(BOUND, async {
tokio::select! {
command = a.recv() => command,
result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()),
}
}).await.unwrap()
else {
panic!("expected second read")
};
@@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits(
}),
);
let mut deletion = Box::pin(f.app.clone().oneshot(request));
assert!(futures::poll!(&mut deletion).is_pending());
let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await;
assert!(
gate.admit(false).is_none(),
"deletion start did not fence admission"
);
tokio::time::timeout(BOUND, async {
loop {
assert!(futures::poll!(&mut deletion).is_pending());
if gate.admit(false).is_none() {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("deletion start did not fence admission after async authentication");
let session_id = session_store::new_session_id().to_string();
let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions");
let callback = f.request(
@@ -57,6 +57,19 @@ impl WorkerOperationContext {
}
Ok(())
}
pub(super) async fn from_request_async(
api: &WorkspaceApi,
context: &server_api::ServerRequestContext,
) -> std::result::Result<Self, server_api::RepositoryApiError> {
let owned_api = api.clone();
let context = context.clone();
api.store
.db_executor()
.run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context)))
.await
.map_err(|error| ApiError::from(error).into_repository_api_error())?
}
pub(super) fn from_request(
api: &WorkspaceApi,
context: &server_api::ServerRequestContext,
@@ -162,8 +175,20 @@ impl<Services> WorkspaceWorker<Services> {
source: crate::worker_source::VerifiedWorkerMutationSource,
reason: &str,
) -> std::result::Result<worker::WorkspaceResponse, String> {
self.current_record()
.map_err(|error| error.error.to_string())?;
let removal = self.removal.clone();
let identity = self.identity.clone();
self.removal
.store
.db_executor()
.run(move || {
removal
.store
.get_worker_registry(&removal.workspace_id, &identity)?
.ok_or_else(|| Error::UnknownWorker { worker: identity })
.map(drop)
})
.await
.map_err(|error| error.to_string())?;
self.removal
.execute_async(
source,
@@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor
}
impl WorkspaceWorker {
pub(crate) async fn resolve_async(
api: &WorkspaceApi,
runtime_id: &str,
reference: &str,
) -> Result<Self> {
let owned_api = api.clone();
let runtime_id = runtime_id.to_owned();
let reference = reference.to_owned();
api.store
.db_executor()
.run(move || Self::resolve(&owned_api, &runtime_id, &reference))
.await
}
pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result<Self> {
Self::resolve_with_services(
api.clone(),
@@ -239,7 +278,36 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
permission: &str,
) -> ApiResult<Option<tokio::sync::OwnedMutexGuard<()>>> {
self.current_record()?;
let worker = self.clone();
let initial_context = context.clone();
let permission = permission.to_owned();
let initial_permission = permission.clone();
let grant = self
.api
.store
.db_executor()
.run(move || -> ApiResult<Option<WorkerControlGrantRecord>> {
worker.current_record()?;
let WorkerOperationContext::WorkerControl { controller } = initial_context else {
return Ok(None);
};
if worker
.api
.store
.get_worker_registry(worker.api.workspace_id(), &controller)?
.is_none()
{
return Err(Error::UnknownWorker { worker: controller }.into());
}
Ok(Some(authorize_known_worker_permission(
&worker.api,
worker.api.workspace_id(),
&controller,
&worker.identity,
&initial_permission,
)?))
})
.await?;
match context {
#[cfg(test)]
WorkerOperationContext::Backend => Ok(None),
@@ -248,34 +316,25 @@ impl WorkspaceWorker {
Ok(None)
}
WorkerOperationContext::WorkerControl { controller } => {
if self
.api
.store
.get_worker_registry(self.api.workspace_id(), controller)?
.is_none()
{
return Err(Error::UnknownWorker {
worker: controller.clone(),
}
.into());
}
let grant = authorize_known_worker_permission(
&self.api,
self.api.workspace_id(),
controller,
&self.identity,
permission,
)?;
let grant = grant.expect("Worker control context resolved a grant");
// Await admission outside the DB lane; the recheck owns the guard
// until it finishes, even if its async waiter is cancelled.
let guard = worker_control_lock(&self.api, &grant.grant_id)
.lock_owned()
.await;
self.current_record()?;
let worker = self.clone();
let controller = controller.clone();
self.api
.store
.db_executor()
.run(move || -> ApiResult<_> {
worker.current_record()?;
let current_grant = authorize_known_worker_permission(
&self.api,
self.api.workspace_id(),
controller,
&self.identity,
permission,
&worker.api,
worker.api.workspace_id(),
&controller,
&worker.identity,
&permission,
)?;
if current_grant.grant_id != grant.grant_id {
return Err(Error::RepositoryConflict(
@@ -284,6 +343,8 @@ impl WorkspaceWorker {
.into());
}
Ok(Some(guard))
})
.await
}
}
}
@@ -293,8 +354,16 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
request: server_api::WorkerRestoreRequest,
) -> ApiResult<InternalWorkerRestoreResult> {
let _guard = self.authorize_operation(context, "restore").await?;
self.restore_internal(request)
let guard = self.authorize_operation(context, "restore").await?;
let worker = self.clone();
self.api
.host_executors
.host()?
.run(move || {
let _guard = guard;
worker.restore_internal(request)
})
.await
}
/// Trusted Backend callers still supply an explicit observed intent. Runtime
@@ -312,27 +381,48 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
request: WorkerLifecycleRequest,
) -> ApiResult<WorkerLifecycleResult> {
let _guard = self.authorize_operation(context, "stop").await?;
let result = self
let guard = self.authorize_operation(context, "stop").await?;
let worker = self.clone();
let (result, guard) = self
.api
.host_executors
.host()?
.run(move || -> ApiResult<_> {
let result = worker
.api
.runtime
.stop_worker(&self.identity, request)
.stop_worker(&worker.identity, request)
.map_err(|e| e.into_error())?;
Ok((result, guard))
})
.await?;
if result.state != InternalWorkerOperationState::Accepted {
return Ok(result);
}
parse_runtime_worker_id_for_registry(&self.identity.worker_id)?;
let session_lock = current_worker_session_lock(&self.api, &self.identity);
let _session_guard = session_lock.lock().await;
let session_guard = session_lock.lock_owned().await;
close_current_worker_session_locked(&self.api, &self.identity).await?;
if let Some(record) = self
let worker = self.clone();
self.api
.host_executors
.host()?
.run(move || -> ApiResult<_> {
let (_control, _session) = (guard, session_guard);
if let Some(record) = worker
.api
.store
.get_worker_registry(self.api.workspace_id(), &self.identity)?
.get_worker_registry(worker.api.workspace_id(), &worker.identity)?
{
sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?;
sync_linked_workdir_after_worker_stop(
&worker.api,
&worker.identity.runtime_id,
&record,
)?;
}
Ok(result)
})
.await
}
pub(super) async fn cancel(
@@ -340,12 +430,20 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
request: WorkerLifecycleRequest,
) -> ApiResult<WorkerLifecycleResult> {
let _guard = self.authorize_operation(context, "cancel").await?;
Ok(self
let guard = self.authorize_operation(context, "cancel").await?;
let worker = self.clone();
self.api
.host_executors
.host()?
.run(move || -> ApiResult<_> {
let _guard = guard;
Ok(worker
.api
.runtime
.cancel_worker(&self.identity, request)
.cancel_worker(&worker.identity, request)
.map_err(|e| e.into_error())?)
})
.await
}
pub(super) async fn input(
@@ -358,29 +456,42 @@ impl WorkspaceWorker {
} else {
"send_input"
};
let _guard = self.authorize_operation(context, permission).await?;
let guard = self.authorize_operation(context, permission).await?;
let session_lock = current_worker_session_lock(&self.api, &self.identity);
let _session_guard = session_lock.lock().await;
self.ensure_interactive()?;
let session_guard = session_lock.lock_owned().await;
self.ensure_interactive().await?;
if matches!(context, WorkerOperationContext::WorkerControl { .. })
&& request.kind == WorkerInputKind::User
{
request.kind = WorkerInputKind::UserIfIdle;
}
Ok(self
let worker = self.clone();
self.api
.host_executors
.host()?
.run(move || -> ApiResult<_> {
let (_control, _session) = (guard, session_guard);
Ok(worker
.api
.runtime
.send_input(&self.identity, request)
.send_input(&worker.identity, request)
.map_err(|e| e.into_error())?)
})
.await
}
fn ensure_interactive(&self) -> ApiResult<()> {
if let Some(claim) = self
async fn ensure_interactive(&self) -> ApiResult<()> {
let worker = self.clone();
self.api
.store
.db_executor()
.run(move || -> ApiResult<()> {
if let Some(claim) = worker
.api
.store
.get_pending_ticket_assignment_operation_for_worker(
self.api.workspace_id(),
&self.identity,
worker.api.workspace_id(),
&worker.identity,
)?
{
return Err(Error::TicketAssignmentConflict(format!(
@@ -388,16 +499,18 @@ impl WorkspaceWorker {
claim.operation_id,
)).into());
}
if self
if worker
.api
.store
.worker_registry_projection(self.api.workspace_id(), &self.identity)?
.worker_registry_projection(worker.api.workspace_id(), &worker.identity)?
.and_then(|projection| projection.job)
.is_some()
{
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
}
Ok(())
})
.await
}
pub(super) async fn set_pinned(
@@ -405,14 +518,23 @@ impl WorkspaceWorker {
context: &WorkerOperationContext,
pinned: bool,
) -> ApiResult<WorkerRetentionResponse> {
let _guard = self.authorize_operation(context, "pin").await?;
let retention_state = if pinned { "pinned" } else { "normal" };
let changed = self.api.worker_projection.publish_ordered(|| {
let guard = self.authorize_operation(context, "pin").await?;
let worker = self.clone();
self.api
.store
.db_executor()
.run(move || -> ApiResult<WorkerRetentionResponse> {
// Publication and its DB mutation are one owned stage. Accepted
// work retains control admission through commit and publication.
let _guard = guard;
let retention_state = if pinned { "pinned" } else { "normal" };
let changed = worker.api.worker_projection.publish_ordered(|| {
worker
.api
.store
.update_worker_retention(
self.api.workspace_id(),
&self.identity,
worker.api.workspace_id(),
&worker.identity,
retention_state,
now_registry_timestamp().as_str(),
)
@@ -420,18 +542,20 @@ impl WorkspaceWorker {
})?;
if !changed {
return Err(cleanup_api_error(
&self.identity.runtime_id,
&worker.identity.runtime_id,
"workspace_worker_retention_unknown_worker",
"Worker is not known to the Backend registry",
));
}
Ok(WorkerRetentionResponse {
workspace_id: self.api.workspace_id().to_owned(),
runtime_id: self.identity.runtime_id.clone(),
worker_id: self.identity.worker_id.clone(),
workspace_id: worker.api.workspace_id().to_owned(),
runtime_id: worker.identity.runtime_id.clone(),
worker_id: worker.identity.worker_id.clone(),
pinned,
retention_state: retention_state.into(),
})
})
.await
}
/// The caller owns the full cleanup-plan digest check. The removal service
@@ -611,7 +735,7 @@ impl WorkspaceWorkerMethodSender {
.await?;
let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity);
let _session_guard = session_lock.lock().await;
self.worker.ensure_interactive()?;
self.worker.ensure_interactive().await?;
let WorkerOperationContext::Browser { source, .. } = context else {
return Err(Error::WorkspacePermissionDenied(
"Browser Worker protocol context required".into(),
@@ -72,6 +72,28 @@ pub(super) struct RequestLease {
callback: bool,
}
impl crate::db_executor::DbOperationLifetime for RequestLease {
fn retain(&self) -> Box<dyn Send> {
// This is continuation of an already admitted request, not fresh admission.
// Retain its drain responsibility before transferring work to the DB owner;
// dropping/aborting the HTTP waiter must not allow deletion past live SQL.
let mut state = self
.gate
.state
.lock()
.expect("Workspace admission poisoned");
if self.callback {
state.callbacks += 1;
} else {
state.requests += 1;
}
Box::new(RequestLease {
gate: self.gate.clone(),
callback: self.callback,
})
}
}
impl Drop for RequestLease {
fn drop(&mut self) {
let mut state = self
@@ -597,7 +597,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) {
let dir = tempfile::tempdir().unwrap();
init_clean_git_workspace(dir.path());
let config = test_server_config(dir.path());
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
seed_test_registered_workspace(store.as_ref(), &config)
.await
.unwrap();
File diff suppressed because it is too large Load Diff
@@ -769,7 +769,7 @@ async fn delivery_real_fixture(
DeliveryScriptedClient,
) {
let config = test_server_config(root);
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
seed_test_registered_workspace(store.as_ref(), &config)
.await
.unwrap();
@@ -1007,7 +1007,7 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
tokio::task::yield_now().await;
let reopened = WorkspaceApi::new_with_execution_backend(
config.clone(),
Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()),
Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()),
execution.clone(),
)
.await
@@ -0,0 +1,769 @@
//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs).
use super::*;
use crate::SqliteWorkspaceStore;
use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime};
use rusqlite::{Connection, ffi};
use std::{
ffi::c_void,
future::Future,
num::NonZeroUsize,
sync::{atomic::AtomicBool, mpsc},
thread::JoinHandle,
time::Duration as StdDuration,
};
use tokio::sync::oneshot;
const WORKSPACE: &str = "db-execution";
const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects";
const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2);
const WATCHDOG: StdDuration = StdDuration::from_secs(5);
async fn observed<T>(future: impl Future<Output = T>) -> T {
tokio::time::timeout(OBSERVE_LIMIT, future)
.await
.expect("request did not reach the expected DB/lifecycle state")
}
struct Fixture {
// Keep temporary authority files alive until all other fixture owners drop.
server: WorkspaceServerApi,
api: WorkspaceApi,
executor: DbExecutor,
_dir: tempfile::TempDir,
}
impl Fixture {
async fn new() -> Self {
let dir = tempfile::tempdir().unwrap();
let executor = DbExecutor::new(DbExecutorLimits {
workers: NonZeroUsize::new(1).unwrap(),
queued: NonZeroUsize::new(1).unwrap(),
})
.unwrap();
// Explicit startup config: never resolve the developer's live data dir.
let config = ServerConfig {
workspace_id: WORKSPACE.into(),
workspace_display_name: "DB execution fixture".into(),
workspace_created_at: "2026-01-01T00:00:00Z".into(),
workspace_execution_root: dir.path().into(),
database_path: dir.path().join("server.db"),
data_root: dir.path().join("data"),
embedded_runtime_store_root: dir.path().join("runtime"),
static_assets_dir: None,
auth: AuthConfig::Passkey {
rp_id: "localhost".into(),
origin: "http://localhost:5173".into(),
public_base_url: "http://localhost:5173".into(),
cookie_name: "db_execution_session".into(),
},
max_records: 100,
repositories: Vec::new(),
runtime_event_sources: Vec::new(),
remote_runtime_sources: Vec::new(),
backend_base_url: Some("http://localhost:8787".into()),
};
let store = Arc::new(
SqliteWorkspaceStore::open_for_test(&config.database_path)
.unwrap()
.with_db_executor(executor.clone()),
);
seed_test_registered_workspace(store.as_ref(), &config)
.await
.unwrap();
store
.upsert_user(&UserRecord {
user_id: "fixture-user".into(),
account_id: format!("account-{WORKSPACE}"),
handle: "fixture-user".into(),
display_name: "Fixture user".into(),
created_at: config.workspace_created_at.clone(),
updated_at: config.workspace_created_at.clone(),
})
.unwrap();
store
.create_browser_session(&crate::store::BrowserSessionRecord {
session_id: "fixture-session".into(),
token_hash: crate::auth::token_hash("fixture-browser-token"),
user_id: "fixture-user".into(),
created_at: config.workspace_created_at.clone(),
expires_at: "2999-01-01T00:00:00Z".into(),
revoked_at: None,
})
.unwrap();
let api = WorkspaceApi::new_for_test(config.clone(), store.clone())
.await
.unwrap();
let server = WorkspaceServerApi::new_for_test(config, store);
// Cache only the real inner router; avoid starting the unrelated hook.
server
.apis
.lock()
.await
.insert(WORKSPACE.into(), api.clone());
server
.routers
.lock()
.await
.insert(WORKSPACE.into(), build_inner_router(api.clone()));
executor.drain().await.unwrap();
Self {
server,
api,
executor,
_dir: dir,
}
}
fn request(method: &str, path: &str, body: serde_json::Value) -> Request {
Request::builder()
.method(method)
.uri(path)
.header(CONTENT_TYPE, "application/json")
.header("cookie", "db_execution_session=fixture-browser-token")
.header(ORIGIN, "http://localhost:5173")
.body(axum::body::Body::from(body.to_string()))
.unwrap()
}
fn subject_request(role: &str) -> Request {
Self::request(
"POST",
SUBJECTS,
serde_json::json!({ "role": role, "behavior_md": "" }),
)
}
fn dispatch(&self, request: Request) -> tokio::task::JoinHandle<Response> {
let server = self.server.clone();
tokio::spawn(async move { dispatch_workspace_request(State(server), request).await })
}
// A pre-authenticated generated router isolates handler admission from the
// earlier browser-session DB read. Full authentication/dispatch is exercised
// separately by the cancellation and warm-Feature tests below.
fn inner_router(&self) -> Router {
build_inner_router(self.api.clone()).layer(Extension(RequestActor {
user_id: "fixture-user".into(),
account_id: format!("account-{WORKSPACE}"),
handle: "fixture-user".into(),
display_name: "Fixture user".into(),
auth_method: ActorAuthMethod::BrowserSession,
}))
}
fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) {
let database = self
.api
.feature_storage
.open(&self.api.subjektiv_registration)
.unwrap();
// Initialize the trusted scope, then obtain SQLite's actual file path
// from the managed handle rather than guessing the Feature directory.
crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap();
let path = database
.with_connection(|conn| {
Ok(PathBuf::from(conn.query_row(
"SELECT file FROM pragma_database_list WHERE name = 'main'",
[],
|row| row.get::<_, String>(0),
)?))
})
.unwrap();
(database, path)
}
}
/// SQLite write ownership lives on an OS thread, including a bounded watchdog.
/// Drop releases and joins it even when an assertion panics on the async side.
struct WriteLock {
release: Option<mpsc::Sender<()>>,
thread: Option<JoinHandle<()>>,
released: Arc<AtomicBool>,
}
impl WriteLock {
fn acquire(path: PathBuf) -> Self {
let (ready, held) = mpsc::sync_channel(1);
let (release, permit) = mpsc::channel();
let released = Arc::new(AtomicBool::new(false));
let flag = released.clone();
let thread = std::thread::spawn(move || {
let conn = Connection::open(path).unwrap();
conn.execute_batch("BEGIN IMMEDIATE").unwrap();
ready.send(()).unwrap();
let _ = permit.recv_timeout(WATCHDOG);
conn.execute_batch("ROLLBACK").unwrap();
flag.store(true, Ordering::SeqCst);
});
held.recv_timeout(WATCHDOG)
.expect("SQLite lock holder did not acquire its transaction");
Self {
release: Some(release),
thread: Some(thread),
released,
}
}
fn release(&mut self) {
if let Some(release) = self.release.take() {
let _ = release.send(());
}
if let Some(thread) = self.thread.take() {
thread.join().unwrap();
}
}
}
impl Drop for WriteLock {
fn drop(&mut self) {
self.release();
}
}
struct BusyGate {
entered: Option<oneshot::Sender<()>>,
resume: mpsc::Receiver<()>,
}
/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's
/// userdata API can, without process globals or changing production hooks.
enum BusyDatabase {
Feature(crate::FeatureDatabase),
Store(Arc<SqliteWorkspaceStore>),
}
impl BusyDatabase {
fn access(&self, access: impl FnOnce(&Connection)) {
match self {
Self::Feature(database) => database
.with_connection(|conn| {
access(conn);
Ok(())
})
.unwrap(),
Self::Store(store) => store
.with_conn(|conn| {
access(conn);
Ok(())
})
.unwrap(),
}
}
}
struct BusyObservation {
database: BusyDatabase,
gate: Box<BusyGate>,
resume: Option<mpsc::Sender<()>>,
}
impl BusyObservation {
fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) {
Self::install_on(BusyDatabase::Feature(database))
}
fn install_store(store: Arc<SqliteWorkspaceStore>) -> (Self, oneshot::Receiver<()>) {
Self::install_on(BusyDatabase::Store(store))
}
fn install_on(database: BusyDatabase) -> (Self, oneshot::Receiver<()>) {
unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 {
// SAFETY: owned Box remains registered until Drop unregisters it,
// serialized by the managed connection mutex.
let gate = unsafe { &mut *context.cast::<BusyGate>() };
if let Some(entered) = gate.entered.take() {
let _ = entered.send(());
return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok());
}
0
}
let (entered, waiting) = oneshot::channel();
let (resume, permit) = mpsc::channel();
let mut gate = Box::new(BusyGate {
entered: Some(entered),
resume: permit,
});
database.access(|conn| {
// SAFETY: handle belongs to this connection; the Box is stable.
let code = unsafe {
ffi::sqlite3_busy_handler(
conn.handle(),
Some(busy),
(&mut *gate as *mut BusyGate).cast(),
)
};
assert_eq!(code, ffi::SQLITE_OK);
});
(
Self {
database,
gate,
resume: Some(resume),
},
waiting,
)
}
fn resume(&mut self) {
if let Some(resume) = self.resume.take() {
let _ = resume.send(());
}
}
}
impl Drop for BusyObservation {
fn drop(&mut self) {
self.resume();
self.database.access(|conn| {
// SAFETY: unregister before the boxed userdata is freed.
unsafe {
ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut());
}
});
let _ = &self.gate;
}
}
// A handler with one DB stage can expose its admission without adding a test
// hook to production. Polling its future to Pending proves run accepted it.
struct AdmissionObservation(Mutex<Option<oneshot::Sender<()>>>);
impl DbOperationLifetime for AdmissionObservation {
fn retain(&self) -> Box<dyn Send> {
if let Some(accepted) = self.0.lock().unwrap().take() {
let _ = accepted.send(());
}
Box::new(())
}
}
fn admission_observation() -> (Arc<dyn DbOperationLifetime>, oneshot::Receiver<()>) {
let (accepted, receiver) = oneshot::channel();
(
Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))),
receiver,
)
}
struct StateCheckSaturation {
executor: DbExecutor,
stages: std::sync::atomic::AtomicUsize,
releases: Arc<Mutex<Vec<mpsc::Sender<()>>>>,
}
impl DbOperationLifetime for StateCheckSaturation {
fn retain(&self) -> Box<dyn Send> {
if self.stages.fetch_add(1, Ordering::SeqCst) == 1 {
// Authentication has completed; saturate just before the lifecycle
// lookup is admitted. The callback's own dispatch admissions are
// distinct stages and do not recurse into this branch.
let (started, running) = mpsc::channel();
let (release, permit) = mpsc::channel();
self.executor
.dispatch(move || {
let _ = started.send(());
let _ = permit.recv_timeout(WATCHDOG);
})
.unwrap();
running
.recv_timeout(WATCHDOG)
.expect("state-check blocker did not start");
self.releases.lock().unwrap().push(release);
let (release, permit) = mpsc::channel();
self.executor
.dispatch(move || {
let _ = permit.recv_timeout(WATCHDOG);
})
.unwrap();
self.releases.lock().unwrap().push(release);
}
Box::new(())
}
}
struct ReleaseStateCheckBlockers(Arc<Mutex<Vec<mpsc::Sender<()>>>>);
impl Drop for ReleaseStateCheckBlockers {
fn drop(&mut self) {
self.0.lock().unwrap().clear();
}
}
#[tokio::test(flavor = "current_thread")]
async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() {
let fixture = Fixture::new().await;
let releases = Arc::new(Mutex::new(Vec::new()));
let release = ReleaseStateCheckBlockers(releases.clone());
let source = Arc::new(StateCheckSaturation {
executor: fixture.executor.clone(),
stages: std::sync::atomic::AtomicUsize::new(0),
releases,
});
let router = Router::new()
.route(
SUBJECTS,
axum::routing::post(|| async { StatusCode::NO_CONTENT }),
)
.layer(axum::middleware::from_fn_with_state(
fixture.api.clone(),
authorize_workspace_api_request,
));
let response = observed(crate::db_executor::with_operation_lifetime(
source,
router.oneshot(Fixture::subject_request("not-executed")),
))
.await
.unwrap();
drop(release);
observed(fixture.executor.drain()).await.unwrap();
let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await;
assert!(
!body.to_string().contains("Workspace is deleting"),
"{body}"
);
}
async fn runtime_progress_while_locked(lock: &WriteLock) {
let control = tokio::spawn(async { "control progressed" });
assert_eq!(observed(control).await.unwrap(), "control progressed");
// The timer is the contract, not a guessed wait for request completion.
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
assert!(
!lock.released.load(Ordering::SeqCst),
"SQLite watchdog released the lock before runtime progress"
);
}
async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value {
let status = response.status();
let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
.await
.unwrap();
assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes));
serde_json::from_slice(&bytes).expect("typed JSON response")
}
fn subject_count(path: &Path, role: &str) -> i64 {
Connection::open(path)
.unwrap()
.query_row(
"SELECT count(*) FROM subjects WHERE role = ?1",
[role],
|row| row.get(0),
)
.unwrap()
}
#[tokio::test(flavor = "current_thread")]
async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() {
let fixture = Fixture::new().await;
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
let (lifetime, accepted) = admission_observation();
let request = Fixture::request(
"POST",
"/api/w/db-execution/tickets",
serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(),
);
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
lifetime,
fixture.inner_router().oneshot(request),
));
assert!(futures::poll!(&mut response).is_pending());
observed(accepted).await.unwrap();
runtime_progress_while_locked(&lock).await;
assert!(
futures::poll!(&mut response).is_pending(),
"mutation responded before the SQLite lock was released"
);
let reader = Connection::open(&fixture.api.config.database_path).unwrap();
let count: i64 = reader
.query_row(
"SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1",
[WORKSPACE],
|row| row.get(0),
)
.unwrap();
assert_eq!(count, 0, "Ticket must not be visible before commit");
lock.release();
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
assert!(body["id"].is_string(), "{body}");
// Ticket dispatch can call Runtime callbacks, so its operation owner is
// the separate bounded host executor, not the store executor.
observed(fixture.api.host_executors.host().unwrap().drain())
.await
.unwrap();
fixture.executor.drain().await.unwrap();
let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap();
assert_eq!(count, 1, "one accepted HTTP mutation must commit once");
}
#[tokio::test(flavor = "current_thread")]
async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
drop(database); // FeatureStorage retains Weak handles: the route must cold-open.
let mut lock = WriteLock::acquire(path);
let (lifetime, accepted) = admission_observation();
let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null);
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
lifetime,
fixture.inner_router().oneshot(request),
));
assert!(futures::poll!(&mut response).is_pending());
observed(accepted).await.unwrap();
runtime_progress_while_locked(&lock).await;
assert!(
futures::poll!(&mut response).is_pending(),
"cold open must wait for its SQLite scope transaction"
);
lock.release();
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
assert_eq!(body["items"], serde_json::json!([]), "{body}");
fixture.executor.drain().await.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
let mut lock = WriteLock::acquire(path.clone());
let (mut busy, waiting) = BusyObservation::install(database);
let response = fixture.dispatch(Fixture::subject_request("locked-subject"));
observed(waiting).await.unwrap();
runtime_progress_while_locked(&lock).await;
assert!(
!response.is_finished(),
"HTTP response must not publish an uncommitted subject"
);
assert_eq!(subject_count(&path, "locked-subject"), 0);
lock.release();
busy.resume();
let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await;
assert_eq!(body["role"], "locked-subject", "{body}");
assert!(
body["memory_fingerprint"].is_string(),
"response assembly must finish in the DB operation: {body}"
);
fixture.executor.drain().await.unwrap();
assert_eq!(subject_count(&path, "locked-subject"), 1);
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
let mut lock = WriteLock::acquire(path.clone());
let (mut busy, waiting) = BusyObservation::install(database);
let response = fixture.dispatch(Fixture::subject_request("cancelled-subject"));
observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling.
response.abort();
assert!(observed(response).await.unwrap_err().is_cancelled());
let admission = fixture.server.admission_for_workspace(WORKSPACE).await;
let mut deletion = Box::pin(admission.lock_deletion());
assert!(
futures::poll!(&mut deletion).is_pending(),
"aborting HTTP must not drop the accepted SQL RequestLease"
);
assert!(
admission.admit(false).is_none(),
"deletion must fence fresh requests"
);
assert_eq!(subject_count(&path, "cancelled-subject"), 0);
lock.release();
busy.resume();
fixture.executor.drain().await.unwrap();
let deletion = observed(deletion).await;
assert_eq!(
subject_count(&path, "cancelled-subject"),
1,
"cancelled waiter must neither cancel nor retry accepted SQL"
);
drop(deletion);
assert!(
admission.admit(false).is_some(),
"deletion lease cleanup must reopen admission"
);
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() {
let fixture = Fixture::new().await;
let (database, path) = fixture.feature_database();
let mut lock = WriteLock::acquire(path.clone());
let (mut busy, waiting) = BusyObservation::install(database);
let running = fixture.dispatch(Fixture::subject_request("running-subject"));
observed(waiting).await.unwrap();
let (lifetime, accepted) = admission_observation();
let mut queued = Box::pin(crate::db_executor::with_operation_lifetime(
lifetime,
fixture
.inner_router()
.oneshot(Fixture::subject_request("queued-subject")),
));
assert!(futures::poll!(&mut queued).is_pending());
observed(accepted).await.unwrap();
drop(queued); // Accepted job still owns the only waiting slot.
let rejected = observed(
fixture
.inner_router()
.oneshot(Fixture::subject_request("rejected-subject")),
)
.await
.unwrap();
let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await;
let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap();
assert!(
error.message.contains("capacity"),
"typed overload error: {body}"
);
assert_eq!(subject_count(&path, "queued-subject"), 0);
assert_eq!(subject_count(&path, "rejected-subject"), 0);
let mut drained = Box::pin(fixture.executor.drain());
assert!(futures::poll!(&mut drained).is_pending());
lock.release();
busy.resume();
response_json(observed(running).await.unwrap(), StatusCode::CREATED).await;
observed(drained).await.unwrap();
assert_eq!(
subject_count(&path, "queued-subject"),
1,
"cancelled accepted mutation persists exactly once"
);
assert_eq!(
subject_count(&path, "rejected-subject"),
0,
"overload must not execute rejected SQL"
);
let retry = fixture.dispatch(Fixture::subject_request("after-drain"));
response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await;
fixture.executor.drain().await.unwrap();
assert_eq!(
subject_count(&path, "after-drain"),
1,
"drain must release actual-job capacity"
);
}
#[tokio::test(start_paused = true)]
async fn saturated_cancelled_host_jobs_keep_their_budget_but_not_other_fixture_or_cleanup_admission()
{
use std::sync::{
Condvar,
atomic::{AtomicUsize, Ordering},
};
// A drop guard releases workers even when an assertion panics.
struct Release(Arc<(Mutex<bool>, Condvar)>);
impl Drop for Release {
fn drop(&mut self) {
*self.0.0.lock().unwrap() = true;
self.0.1.notify_all();
}
}
let owner = HostExecutors::default();
let other = HostExecutors::default();
let host = owner.host().unwrap();
let gate = Arc::new((Mutex::new(false), Condvar::new()));
let release = Release(gate.clone());
let completed = Arc::new(AtomicUsize::new(0));
let mut waiters = Vec::new();
for _ in 0..2 {
let host = host.clone();
let gate = gate.clone();
let completed = completed.clone();
let (started, accepted) = tokio::sync::oneshot::channel();
waiters.push(tokio::spawn(async move {
host.run(move || {
started.send(()).unwrap();
let mut released = gate.0.lock().unwrap();
while !*released {
released = gate.1.wait(released).unwrap();
}
completed.fetch_add(1, Ordering::SeqCst);
Ok::<_, Error>(())
})
.await
}));
accepted.await.unwrap();
}
for _ in 0..16 {
let completed = completed.clone();
host.dispatch(move || {
completed.fetch_add(1, Ordering::SeqCst);
})
.unwrap();
}
for waiter in waiters {
waiter.abort();
assert!(waiter.await.unwrap_err().is_cancelled());
}
assert!(matches!(
host.run(|| Ok::<_, Error>(())).await,
Err(Error::DbCapacityExceeded)
));
// Paused Tokio time can advance without completing or releasing actual Host
// work. Neither another fixture nor cleanup needs this saturated Host lane.
tokio::time::advance(std::time::Duration::from_secs(31)).await;
assert_eq!(completed.load(Ordering::SeqCst), 0);
other
.host()
.unwrap()
.run(|| Ok::<_, Error>(()))
.await
.unwrap();
owner
.cleanup()
.unwrap()
.run(|| Ok::<_, Error>(()))
.await
.unwrap();
assert!(matches!(
host.run(|| Ok::<_, Error>(())).await,
Err(Error::DbCapacityExceeded)
));
drop(release);
host.drain().await.unwrap();
assert_eq!(completed.load(Ordering::SeqCst), 18);
host.run(|| Ok::<_, Error>(())).await.unwrap();
host.drain().await.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn restore_intent_pins_once_after_write_ownership_without_a_deferred_read_upgrade() {
let fixture = Fixture::new().await;
let store = fixture.api.config_store.clone();
let (mut busy, waiting) = BusyObservation::install_store(store.clone());
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
let worker = RuntimeWorkerRef::new("restore-runtime", "restore-worker");
let pin_store = store.clone();
let pin_worker = worker.clone();
let mut pinned = Box::pin(fixture.executor.run(move || {
pin_store.pin_internal_worker_restore_intent(
WORKSPACE,
&pin_worker,
"restore-domain",
Some("observed-before-dispatch"),
Some("restore-request"),
)
}));
tokio::select! {
result = &mut pinned => panic!("restore intent must wait for write ownership before reading, not fail a deferred read upgrade: {result:?}"),
arrived = observed(waiting) => arrived.unwrap(),
}
runtime_progress_while_locked(&lock).await;
assert!(futures::poll!(&mut pinned).is_pending());
lock.release();
busy.resume();
let request = observed(pinned).await.unwrap();
assert_eq!(request.request_id, "restore-request");
assert_eq!(
request.expected_observation_token,
"observed-before-dispatch"
);
fixture.executor.drain().await.unwrap();
let saved = store
.get_internal_worker_restore_intent(WORKSPACE, &worker, &request.request_id)
.unwrap()
.unwrap();
assert_eq!(saved.request_id, request.request_id);
assert_eq!(
saved.expected_observation_token,
request.expected_observation_token
);
let count: i64 = store
.with_conn(|conn| {
Ok(conn.query_row(
"SELECT count(*) FROM worker_restore_intents WHERE workspace_id=?1 AND request_id=?2",
rusqlite::params![WORKSPACE, request.request_id], |row| row.get(0)
)?)
})
.unwrap();
assert_eq!(
count, 1,
"one admitted intent must persist exactly once, with no operation retry"
);
}
+6 -7
View File
@@ -51,14 +51,13 @@ async fn blocking<T: Send + 'static>(
mutation: bool,
action: impl FnOnce() -> workspace_drive::Result<T> + Send + 'static,
) -> DriveResult<T> {
tokio::task::spawn_blocking(action)
crate::db_executor::DbExecutor::shared()
.run(move || Ok::<_, Error>(action()))
.await
.map_err(|_| {
if mutation {
DriveApiError::new(DriveApiErrorCode::OutcomeUnknown)
} else {
failure(503)
}
.map_err(|error| match error {
Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503),
_ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown),
_ => failure(503),
})?
.map_err(|error| domain_error(error, mutation))
}
@@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> DriveApiError {
})
}
fn mutation_error(error: Error) -> DriveApiError {
if matches!(error, Error::Sqlite(_) | Error::Store(_)) {
if matches!(
error,
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
) {
fail(DriveApiErrorCode::OutcomeUnknown)
} else {
pre_error(error)
@@ -97,9 +100,13 @@ pub(super) async fn authorized_drive(
};
// Reject missing/read-only grants before upload body reception. This check
// is admission only: every later operation rechecks under its own lock.
tokio::task::spawn_blocking(move || {
api.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
drive.check_authorized(write)?;
Ok::<_, workspace_drive::Error>(drive)
})())
})
.await
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
@@ -152,9 +159,10 @@ pub(super) async fn create_grant(
revoked_at: None,
};
let store = api.store.clone();
tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant))
api.store
.db_executor()
.run(move || store.create_workspace_drive_grant(&grant))
.await
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
.map_err(mutation_error)
}
@@ -175,11 +183,10 @@ pub(super) async fn revoke_grant(
actor.account_id.clone(),
);
let store = api.store.clone();
tokio::task::spawn_blocking(move || {
store.revoke_workspace_drive_grant(&workspace, &grant, &actor)
})
api.store
.db_executor()
.run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor))
.await
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
.map_err(mutation_error)?
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
}
@@ -212,11 +219,13 @@ pub(super) async fn list_grants(
}
let workspace = workspace_id.to_owned();
let store = api.store.clone();
let mut grants = tokio::task::spawn_blocking(move || {
let mut grants = api
.store
.db_executor()
.run(move || {
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
})
.await
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
.map_err(pre_error)?;
let more = grants.len() > limit;
grants.truncate(limit);
@@ -724,7 +724,7 @@ async fn catalog_case_dispatch_authenticates_and_routes_the_canonical_workspace_
reason: ticket::TicketClosureReason::Completed,
},
);
let catalog = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
let catalog = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
// Retain only this in-process fixture: do not create a real Runtime or a
// background Orchestrator hook while testing catalog authentication/routing.
catalog
@@ -529,7 +529,7 @@ async fn runtime_tag_mutations_reject_invalid_proofs_and_changed_binding_authori
*tampered.body_mut() = Body::from(format!("{} ", input));
runtime_tag_response(app.clone(), tampered, StatusCode::UNAUTHORIZED).await;
}
let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap();
let original = api
.store
.get_workspace_runtime_binding(TEST_WORKSPACE_ID, "runtime-tags")
@@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> Failure {
fn mutation_error(error: Error) -> Failure {
// Storage/service failures at a mutation boundary may include a commit
// acknowledgement failure. Never describe those as a guaranteed rollback.
if matches!(&error, Error::Sqlite(_) | Error::Store(_)) {
if matches!(
&error,
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
) {
unknown()
} else {
pre_error(error)
@@ -186,9 +189,23 @@ pub(super) async fn get(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
) -> ConfigResult<Option<WorkspaceConfigAttachment>> {
let _guard = lock(api, worker).await?;
let grant = grant_for_worker(api, worker)?;
Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true)))
let guard = lock(api, worker).await?;
let executor = api.store.db_executor();
let api = api.clone();
let worker = worker.clone();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
let grant = grant_for_worker(&api, &worker)?;
Ok::<_, Failure>(
active_link(&api, &worker, &grant, None)?
.map(|link| attachment(&grant, &link, true)),
)
})())
})
.await
.map_err(pre_error)?
}
pub(super) async fn attach(
api: &WorkspaceApi,
@@ -198,20 +215,29 @@ pub(super) async fn attach(
if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) {
return Err(fail(400, "invalid_alias"));
}
let _guard = lock(api, worker).await?;
let grant = grant_for_worker(api, worker)?;
let guard = lock(api, worker).await?;
let executor = api.store.db_executor();
let api = api.clone();
let worker = worker.clone();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
let grant = grant_for_worker(&api, &worker)?;
let access = request.access.unwrap_or(grant.access);
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
return Err(fail(403, "access_denied"));
}
if let Some(mut link) = active_link(api, worker, &grant, None)? {
if let Some(mut link) = active_link(&api, &worker, &grant, None)? {
// Same active alias/identity is idempotent; explicit write never silently downgrades.
if request.access == Some(Access::ReadWrite)
&& effective_access(&grant, &link) != Access::ReadWrite
{
return Err(fail(403, "access_denied"));
}
if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly {
if access == Access::ReadOnly
&& effective_access(&grant, &link) != Access::ReadOnly
{
link.capabilities = access_capabilities(access);
link = api
.store
@@ -235,9 +261,13 @@ pub(super) async fn attach(
.map_err(mutation_error)?;
// No Runtime filesystem/session synchronization for a logical config target.
api.worker_projection
.refresh(worker)
.refresh(&worker)
.map_err(|_| unknown())?;
Ok(attachment(&grant, &link, false))
})())
})
.await
.map_err(mutation_error)?
}
fn resolve(
api: &WorkspaceApi,
@@ -313,9 +343,22 @@ pub(super) async fn observe(
for path in &request.paths {
path_valid(path, true)?;
}
let _guard = lock(api, worker).await?;
let (grant, link) = resolve(api, worker, &request.connection_id)?;
let state = tree(api)?;
let guard = lock(api, worker).await?;
let resolution_api = api.clone();
let resolution_worker = worker.clone();
let connection_id = request.connection_id.clone();
let (grant, link, state, _guard) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
let state = tree(&resolution_api)?;
Ok::<_, Failure>((grant, link, state, guard))
})())
})
.await
.map_err(pre_error)??;
let schema_fingerprint = api
.config_schema_registry
.compose()
@@ -427,9 +470,22 @@ pub(super) async fn read(
request: WorkspaceConfigReadRequest,
) -> ConfigResult<WorkspaceConfigReadResponse> {
path_valid(&request.path, false)?;
let _guard = lock(api, worker).await?;
let (grant, link) = resolve(api, worker, &request.connection_id)?;
let state = tree(api)?;
let guard = lock(api, worker).await?;
let resolution_api = api.clone();
let resolution_worker = worker.clone();
let connection_id = request.connection_id.clone();
let (grant, link, state, _guard) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
let state = tree(&resolution_api)?;
Ok::<_, Failure>((grant, link, state, guard))
})())
})
.await
.map_err(pre_error)??;
let schema_fingerprint = api
.config_schema_registry
.compose()
@@ -482,12 +538,25 @@ pub(super) async fn commit(
server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?,
}
}
let _guard = lock(api, worker).await?;
let (grant, link) = resolve(api, worker, &request.connection_id)?;
let guard = lock(api, worker).await?;
let resolution_api = api.clone();
let resolution_worker = worker.clone();
let connection_id = request.connection_id.clone();
let (grant, link, state, guard) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
if effective_access(&grant, &link) != Access::ReadWrite {
return Err(fail(403, "access_denied"));
}
let state = tree(api)?;
let state = tree(&resolution_api)?;
Ok::<_, Failure>((grant, link, state, guard))
})())
})
.await
.map_err(pre_error)??;
let schema_fingerprint = api
.config_schema_registry
.compose()
@@ -501,17 +570,19 @@ pub(super) async fn commit(
// Evaluation/semantic validation are pure preparation. Bound this stage
// without ever dispatching persistence in a timed-out background task.
let preparation_api = api.clone();
let preparation = tokio::task::spawn_blocking(move || {
prepare_workspace_config_tree(
let executor = api.store.db_executor();
let preparation = executor.run(move || {
// Timeout/cancellation releases exclusion only after actual preparation ends.
let candidate = prepare_workspace_config_tree(
&preparation_api,
&preparation_api.config.workspace_id,
&canonical,
)
)?;
Ok::<_, Error>((candidate, guard))
});
let candidate = tokio::time::timeout(DEADLINE, preparation)
let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation)
.await
.map_err(|_| fail(503, "evaluation_timeout"))?
.map_err(|_| fail(500, "evaluation_unavailable"))?
.map_err(pre_error)?;
if candidate.base_digest != state.snapshot.digest
|| candidate.base_toolchain_fingerprint != state.contract.fingerprint
@@ -520,15 +591,78 @@ pub(super) async fn commit(
{
return Err(fail(409, "stale_validator"));
}
// Recheck durable authority before the first auxiliary/canonical effects.
let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?;
if current_grant != grant || current_link.capabilities != link.capabilities {
let persistence_api = api.clone();
let worker = worker.clone();
let connection_id = request.connection_id.clone();
let expected_grant = grant.clone();
let expected_link = link.clone();
let (state, publication) = api
.store
.db_executor()
.run(move || {
Ok::<_, Error>((|| {
let (current_grant, current_link) =
resolve(&persistence_api, &worker, &connection_id)?;
if current_grant != expected_grant
|| current_link.capabilities != expected_link.capabilities
{
return Err(fail(409, "authority_changed"));
}
// From this boundary repository credential projection may have side effects.
// Never falsely claim rollback or automatically replay a failed/uncertain save.
let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate)
// Preserve the shared helper's ordering and atomic CAS, but never
// run its remote Runtime publication on the sole DB worker.
project_repository_access_candidate(
&*persistence_api.store,
&persistence_api.repository_secrets,
&persistence_api.config.workspace_id,
&candidate,
)
.map_err(|_| unknown())?;
let state = persistence_api
.config_store
.commit_evaluated_workspace_config(
&persistence_api.config.workspace_id,
&candidate,
)
.map_err(|_| unknown())?;
let publication = persistence_api
.prompt_projection_cache
.resolve(&persistence_api.config.workspace_id, &state)
.ok()
.and_then(|projection| {
// Dispatch from the actual persistence owner even if its
// waiter was cancelled. Runtime I/O owns exclusion, not
// a DB worker or an open SQL transaction.
let (complete, publication) = tokio::sync::oneshot::channel();
let runtime = persistence_api.runtime.clone();
let admission = persistence_api.host_executors.host().and_then(|executor| {
executor.dispatch(move || {
let _guard = guard;
let _diagnostics = runtime
.observe_workspace_prompt_projection((*projection).clone());
let _ = complete.send(());
})
});
match admission {
Ok(()) => Some(publication),
Err(error) => {
tracing::warn!(
workspace_id = %persistence_api.config.workspace_id,
%error,
"Workspace config committed but Runtime publication was not admitted"
);
None
}
}
});
Ok((state, publication))
})())
})
.await
.map_err(mutation_error)??;
if let Some(publication) = publication {
// Publication is best effort and cannot turn an acknowledged CAS into an error.
let _ = publication.await;
}
Ok(WorkspaceConfigCommitResponse {
validator: validator(&grant, &link, &state, &schema_fingerprint, ""),
digest: state.snapshot.digest,
@@ -544,17 +678,37 @@ pub(super) async fn create_grant(
.await
.map_err(pre_error)?;
let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id);
let _guard = lock(api, &worker).await?;
let summary = api
.runtime
.worker(&worker)
let guard = lock(api, &worker).await?;
let runtime = api.runtime.clone();
let lookup_worker = worker.clone();
let (summary, guard) = api
.host_executors
.host()
.map_err(pre_error)?
.run(move || {
Ok::<_, Error>((|| {
let summary = runtime
.worker(&lookup_worker)
.map_err(|_| fail(404, "worker_not_found"))?;
Ok::<_, Failure>((summary, guard))
})())
})
.await
.map_err(pre_error)??;
if summary.workspace.workspace_id.as_deref() != Some(workspace_id) {
return Err(fail(403, "access_denied"));
}
let executor = api.store.db_executor();
let api = api.clone();
let workspace_id = workspace_id.to_owned();
let account_id = actor.account_id.clone();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
if let Some(grant) = api
.store
.current_workspace_config_grant(workspace_id, &worker)
.current_workspace_config_grant(&workspace_id, &worker)
.map_err(pre_error)?
{
if grant.access != request.access {
@@ -564,7 +718,7 @@ pub(super) async fn create_grant(
}
let grant = WorkspaceConfigGrantResponse {
grant_id: Uuid::now_v7().to_string(),
workspace_id: workspace_id.into(),
workspace_id,
runtime_id: worker.runtime_id,
worker_id: worker.worker_id,
working_directory_id: Uuid::now_v7().to_string(),
@@ -572,9 +726,13 @@ pub(super) async fn create_grant(
revoked: false,
};
api.store
.create_workspace_config_grant(&grant, &actor.account_id)
.create_workspace_config_grant(&grant, &account_id)
.map_err(mutation_error)?;
Ok(grant)
})())
})
.await
.map_err(mutation_error)?
}
pub(super) async fn revoke_grant(
api: &WorkspaceApi,
@@ -585,28 +743,45 @@ pub(super) async fn revoke_grant(
require_workspace_owner(api, workspace_id, actor, "Workspace config grants")
.await
.map_err(pre_error)?;
let lookup_api = api.clone();
let lookup_workspace_id = workspace_id.to_owned();
let lookup_grant_id = grant_id.to_owned();
let mut grant = api
.store
.get_workspace_config_grant(workspace_id, grant_id)
.db_executor()
.run(move || {
lookup_api
.store
.get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id)
})
.await
.map_err(pre_error)?
.ok_or_else(|| fail(404, "not_found"))?;
let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id);
let _guard = lock(api, &worker).await?;
let guard = lock(api, &worker).await?;
let executor = api.store.db_executor();
let api = api.clone();
let workspace_id = workspace_id.to_owned();
let grant_id = grant_id.to_owned();
executor
.run(move || {
let _guard = guard;
Ok::<_, Error>((|| {
api.store
.revoke_workspace_config_grant(workspace_id, grant_id)
.revoke_workspace_config_grant(&workspace_id, &grant_id)
.map_err(mutation_error)?;
grant.revoked = true;
// Revocation is durable before unlink; a failed cleanup cannot restore access.
if let Some(link) = api
.store
.list_workdir_worker_links(workspace_id, &grant.working_directory_id)
.list_workdir_worker_links(&workspace_id, &grant.working_directory_id)
.map_err(|_| unknown())?
.into_iter()
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
{
api.store
.detach_worker_workdir_connection(
workspace_id,
&workspace_id,
&worker,
&link.alias,
&link.connection_id,
@@ -618,6 +793,10 @@ pub(super) async fn revoke_grant(
.refresh(&worker)
.map_err(|_| unknown())?;
Ok(grant)
})())
})
.await
.map_err(mutation_error)?
}
pub(super) fn summary(
@@ -655,7 +834,11 @@ pub(super) fn summary(
mod tests {
use super::*;
#[test]
fn workspace_config_mutation_storage_failures_are_conservatively_unknown() {
fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() {
assert_eq!(
mutation_error(Error::DbOperationOutcomeUnknown).classification,
Classification::Unknown
);
assert_eq!(
mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification,
Classification::Unknown
@@ -318,7 +318,7 @@ mod workspace_config_integration {
};
for scoped in [false, true] {
let app = if scoped {
workspace_server_router(WorkspaceServerApi::new(
workspace_server_router(WorkspaceServerApi::new_for_test(
api.config.clone(),
api.store.clone(),
))
@@ -803,9 +803,9 @@ mod workspace_config_integration {
.supports(workdir::WorkdirSessionCapability::Command)
);
let lock = current_worker_session_lock(api, &fixture.worker);
let _guard = lock.lock().await;
let _guard = Arc::new(lock.lock_owned().await);
assert!(
open_current_worker_workdir_session_locked(api, &fixture.worker, &link)
open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard)
.await
.is_err()
);
@@ -1385,7 +1385,7 @@ mod workspace_config_integration {
let api = &fixture.api;
let granted = grant(api, &fixture.worker, Access::ReadWrite).await;
let attached = attach(api, &fixture.worker).await;
let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap();
let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap();
assert_eq!(
restarted
.get_workspace_config_grant(TEST_WORKSPACE_ID, &granted.grant_id)
@@ -800,7 +800,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
let base = format!("http://{addr}");
let mut config = test_server_config(root.path());
config.backend_base_url = Some("server-test".into());
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
let captured = Arc::new(Mutex::new(None));
let abort = Arc::new(Mutex::new(None));
let factory = ConfigCaptureFactory {
File diff suppressed because it is too large Load Diff
@@ -263,7 +263,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64
let tags_before = retained.as_ref().map(|_| classification_rows(&conn));
let auth_before = fixture_auth_rows(&conn);
drop(conn);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?,93);
assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11);
@@ -490,7 +490,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history
let metadata_before = stored_fixture_rows(&conn, "typed_ticket_case_metadata");
let auth_before = fixture_auth_rows(&conn);
drop(conn);
let store = SqliteWorkspaceStore::open(&path).unwrap();
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store
.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?,93);
@@ -750,7 +750,7 @@ mod tests {
#[test]
fn retry_keeps_resolved_config_and_credential_candidates_after_fallback_moves() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace".to_string(),
owner_account_id: "owner-account".to_string(),
@@ -925,7 +925,7 @@ mod tests {
}
async fn seeded_store() -> (SqliteWorkspaceStore, WorkdirRegistryRecord) {
let store = SqliteWorkspaceStore::in_memory().unwrap();
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
store
.upsert_account(&AccountRecord {
account_id: "account-a".to_string(),
+433 -20
View File
@@ -5,7 +5,7 @@ use chrono::Utc;
use protocol::subscription::{
EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot,
};
use tokio::sync::broadcast;
use tokio::sync::{Semaphore, broadcast};
use tokio::task::JoinHandle;
use crate::runtime_subscription::{
@@ -59,7 +59,14 @@ pub struct WorkerProjectionService {
events: broadcast::Sender<WorkerProjectionEvent>,
ordering: Mutex<()>,
attach_gate: tokio::sync::Mutex<()>,
tasks: Mutex<HashMap<String, JoinHandle<()>>>,
tasks: Mutex<HashMap<String, RuntimeProjectionLifetime>>,
}
struct RuntimeProjectionLifetime {
consumer: Option<JoinHandle<()>>,
// The permit belongs to the actual DB closure, not its cancellable async waiter.
// Keep this owner in the map even if attachment initialization is cancelled.
operations: Arc<Semaphore>,
}
impl WorkerProjectionService {
@@ -75,16 +82,88 @@ impl WorkerProjectionService {
})
}
/// Stops consumers but does not claim that accepted database work has finished.
/// Async shutdown callers must use `shutdown_async` to drain accepted operations.
pub fn shutdown(&self) {
for (_, task) in self
for lifetime in self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.drain()
.values()
{
if let Some(task) = &lifetime.consumer {
task.abort();
}
}
}
pub async fn shutdown_async(&self) {
let _attach = self.attach_gate.lock().await;
self.shutdown();
let runtime_ids = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.keys()
.cloned()
.collect::<Vec<_>>();
for runtime_id in runtime_ids {
self.drain_runtime(&runtime_id).await;
}
self.tasks
.lock()
.expect("Worker projection task lock poisoned")
.clear();
}
async fn drain_runtime(&self, runtime_id: &str) {
let previous = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.get_mut(runtime_id)
.map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone()));
if let Some((consumer, operations)) = previous {
if let Some(consumer) = consumer {
consumer.abort();
let _ = consumer.await;
}
// Aborting the waiter cannot abort a queued/running synchronous job.
// Cancellation here leaves the gate in the map for the next drain.
let _drained = operations
.acquire_owned()
.await
.expect("operation gate closed");
}
}
// Caller holds attach_gate. Install ownership before any initialization await.
async fn replace_runtime(&self, runtime_id: &str) -> Arc<Semaphore> {
self.drain_runtime(runtime_id).await;
let operations = Arc::new(Semaphore::new(1));
self.tasks
.lock()
.expect("Worker projection task lock poisoned")
.insert(
runtime_id.to_string(),
RuntimeProjectionLifetime {
consumer: None,
operations: operations.clone(),
},
);
operations
}
pub async fn subscribe_ordered_async(
self: &Arc<Self>,
limit: usize,
) -> crate::Result<WorkerProjectionSubscription> {
let service = self.clone();
self.store
.db_executor()
.run(move || service.subscribe_ordered(limit))
.await
}
pub fn subscribe_ordered(&self, limit: usize) -> crate::Result<WorkerProjectionSubscription> {
let _ordering = self
@@ -125,19 +204,10 @@ impl WorkerProjectionService {
broker: &RuntimeSubscriptionBroker,
runtime_id: &str,
) -> Result<(), RuntimeSubscriptionBrokerError> {
// Serialize replacement and wait for the old consumer to stop before the new
// lifetime can apply its snapshot. Late notifications can therefore never
// cross a Runtime ownership boundary.
// Serialize replacement, stop admission by aborting the consumer, and then
// drain ownership held by the actual SQL jobs before the new snapshot.
let _attach = self.attach_gate.lock().await;
let previous = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.remove(runtime_id);
if let Some(previous) = previous {
previous.abort();
let _ = previous.await;
}
let operations = self.replace_runtime(runtime_id).await;
let mut subscription =
broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?;
@@ -148,12 +218,25 @@ impl WorkerProjectionService {
break;
};
let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. });
if let Err(error) = self.apply_broker_event(runtime_id, event) {
if let Err(error) = self
.apply_broker_event_owned_async(runtime_id, event, operations.clone())
.await
{
tracing::warn!(
runtime_id,
error = %error,
"failed to initialize durable Worker projection; restarting from a fresh snapshot"
);
if matches!(
error,
crate::Error::DbOperationOutcomeUnknown
| crate::Error::DbExecutorUnavailable
| crate::Error::DbCapacityExceeded
) {
// Do not automatically replay a possibly committed mutation or
// spin on executor overload; attachment must be retried explicitly.
return Err(RuntimeSubscriptionBrokerError::Closed);
}
subscription.restart()?;
continue;
}
@@ -169,12 +252,27 @@ impl WorkerProjectionService {
let Some(service) = service.upgrade() else {
return;
};
if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) {
if let Err(error) = service
.apply_broker_event_owned_async(
task_runtime_id.as_str(),
event,
operations.clone(),
)
.await
{
tracing::warn!(
runtime_id = %task_runtime_id,
error = %error,
"failed to update durable Worker projection; restarting from a fresh snapshot"
);
if matches!(
error,
crate::Error::DbOperationOutcomeUnknown
| crate::Error::DbExecutorUnavailable
| crate::Error::DbCapacityExceeded
) {
return;
}
if subscription.restart().is_err() {
return;
}
@@ -184,10 +282,61 @@ impl WorkerProjectionService {
self.tasks
.lock()
.expect("Worker projection task lock poisoned")
.insert(runtime_id, task);
.get_mut(&runtime_id)
.expect("Runtime lifetime installed before initialization")
.consumer = Some(task);
Ok(())
}
#[cfg(test)]
async fn seed_observation_async(
self: &Arc<Self>,
runtime_id: &str,
worker: &protocol::subscription::SubscriptionWorker,
observed_at: &str,
) -> crate::Result<()> {
let service = self.clone();
let runtime_id = runtime_id.to_string();
let worker = worker.clone();
let observed_at = observed_at.to_string();
self.store
.db_executor()
.run(move || service.seed_observation(&runtime_id, &worker, &observed_at))
.await
}
pub async fn refresh_async(self: &Arc<Self>, worker: &RuntimeWorkerRef) -> crate::Result<()> {
let service = self.clone();
let worker = worker.clone();
self.store
.db_executor()
.run(move || service.refresh(&worker))
.await
}
async fn apply_broker_event_owned_async(
self: &Arc<Self>,
runtime_id: &str,
event: BrokerSubscriptionEvent,
operations: Arc<Semaphore>,
) -> crate::Result<()> {
let operation = operations
.acquire_owned()
.await
.expect("operation gate closed");
let service = self.clone();
let runtime_id = runtime_id.to_string();
self.store
.db_executor()
.run(move || {
// The permit must outlive SQL commit AND ordered publication, even
// when the executor result receiver has been dropped by cancellation.
let _operation = operation;
service.apply_broker_event(&runtime_id, event)
})
.await
}
pub fn seed_observation(
&self,
runtime_id: &str,
@@ -313,7 +462,11 @@ mod tests {
tempfile::TempDir,
) {
let temp = tempfile::tempdir().unwrap();
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap());
let store = Arc::new(
SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db"))
.unwrap()
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
);
store
.upsert_account(&AccountRecord {
account_id: "owner".to_string(),
@@ -355,6 +508,266 @@ mod tests {
}
}
fn observation(
state: protocol::subscription::SubscriptionWorkerState,
) -> protocol::subscription::SubscriptionWorker {
protocol::subscription::SubscriptionWorker {
restore_observation_token: None,
worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(),
runtime_id: Some("runtime-a".to_string()),
resource_key: None,
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
worker_state: None,
state,
has_running_internal_workers: false,
workspace_id: Some(WORKSPACE_ID.to_string()),
display_name: None,
profile: None,
job: None,
workdir_attachments: Vec::new(),
}
}
// Real second-connection contention; the controller releases automatically on
// failure, so a regression that blocks the Tokio thread cannot hang the suite.
struct WriteLock {
release: Option<std::sync::mpsc::Sender<()>>,
controller: Option<std::thread::JoinHandle<()>>,
}
impl WriteLock {
fn acquire(path: std::path::PathBuf) -> Self {
let (release, released) = std::sync::mpsc::channel();
let (locked, ready) = std::sync::mpsc::channel();
let controller = std::thread::spawn(move || {
let connection = rusqlite::Connection::open(path).unwrap();
connection.execute_batch("BEGIN IMMEDIATE").unwrap();
locked.send(()).unwrap();
let _ = released.recv_timeout(std::time::Duration::from_secs(3));
connection.execute_batch("ROLLBACK").unwrap();
});
ready.recv().unwrap();
Self {
release: Some(release),
controller: Some(controller),
}
}
fn release(&mut self) {
if let Some(release) = self.release.take() {
let _ = release.send(());
}
}
}
impl Drop for WriteLock {
fn drop(&mut self) {
self.release();
if let Some(controller) = self.controller.take() {
controller.join().unwrap();
}
}
}
async fn wait_for_sql(service: &WorkerProjectionService) {
tokio::time::timeout(std::time::Duration::from_secs(1), async {
// With the second connection holding the write lock, a held ordering
// gate identifies the accepted synchronous mutation, not a fake future.
while service.ordering.try_lock().is_ok() {
tokio::task::yield_now().await;
}
})
.await
.expect("projection job did not enter synchronous SQL");
}
#[tokio::test(flavor = "current_thread")]
async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload()
{
use protocol::subscription::SubscriptionWorkerState;
let (store, service, temp) = fixture().await;
store
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
.unwrap();
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
assert_eq!(subscription.take_snapshot().len(), 1);
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
let publishing_service = service.clone();
let publication = tokio::spawn(async move {
publishing_service
.seed_observation_async(
"runtime-a",
&observation(SubscriptionWorkerState::Idle),
"2",
)
.await
});
wait_for_sql(&service).await;
let started = std::time::Instant::now();
let (control, controlled) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
control.send("control progressed").unwrap();
});
let (_, control_result) = tokio::join!(
tokio::time::sleep(std::time::Duration::from_millis(20)),
controlled,
);
assert_eq!(control_result.unwrap(), "control progressed");
assert!(
started.elapsed() < std::time::Duration::from_secs(1),
"SQLite blocked the current-thread executor"
);
assert!(matches!(
subscription.try_recv(),
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
));
lock.release();
publication.await.unwrap().unwrap();
let committed = subscription.recv().await.unwrap();
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
let snapshot = snapshot.take_snapshot();
assert_eq!(
committed.changes,
vec![WorkerCatalogChange::Upsert(snapshot[0].clone())]
);
assert_eq!(
snapshot[0].observation.as_ref().unwrap().worker.state,
SubscriptionWorkerState::Idle
);
service
.seed_observation_async(
"runtime-a",
&observation(SubscriptionWorkerState::Running),
"3",
)
.await
.unwrap();
// Delayed readers receive the first transaction's payload, not a re-query
// of mutable authority after a later commit.
let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else {
panic!("expected upsert");
};
assert_eq!(
first.observation.as_ref().unwrap().worker.state,
SubscriptionWorkerState::Idle
);
service
.refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a"))
.await
.unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() {
use crate::runtime_subscription::BrokerSubscriptionEvent;
use protocol::subscription::SubscriptionWorkerState;
let (store, service, temp) = fixture().await;
store
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
.unwrap();
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
subscription.take_snapshot();
let operations = service.replace_runtime("runtime-a").await;
let old_operations = operations.clone();
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
let old_service = service.clone();
let consumer = tokio::spawn(async move {
old_service
.apply_broker_event_owned_async(
"runtime-a",
BrokerSubscriptionEvent::Event {
payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted {
worker: observation(SubscriptionWorkerState::Running),
},
},
operations,
)
.await
.unwrap();
});
service
.tasks
.lock()
.unwrap()
.get_mut("runtime-a")
.unwrap()
.consumer = Some(consumer);
wait_for_sql(&service).await;
let replacing_service = service.clone();
let mut replacement = tokio::spawn(async move {
let _attach = replacing_service.attach_gate.lock().await;
replacing_service.replace_runtime("runtime-a").await
});
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
.await
.is_err()
);
replacement.abort();
assert!(replacement.await.unwrap_err().is_cancelled());
assert_eq!(
old_operations.available_permits(),
0,
"cancelled drain lost accepted SQL ownership"
);
// A second replacement must still observe the old gate after cancellation.
let replacing_service = service.clone();
let mut replacement = tokio::spawn(async move {
let _attach = replacing_service.attach_gate.lock().await;
let operations = replacing_service.replace_runtime("runtime-a").await;
replacing_service
.apply_broker_event_owned_async(
"runtime-a",
BrokerSubscriptionEvent::Snapshot {
snapshot: protocol::subscription::SubscriptionSnapshot::Workers {
workers: vec![observation(SubscriptionWorkerState::Idle)],
},
},
operations,
)
.await
});
assert!(
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
.await
.is_err()
);
assert!(matches!(
subscription.try_recv(),
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
));
lock.release();
replacement.await.unwrap().unwrap();
assert_eq!(old_operations.available_permits(), 1);
let old_commit = subscription.recv().await.unwrap();
let new_commit = subscription.recv().await.unwrap();
for (commit, expected) in [
(old_commit, SubscriptionWorkerState::Running),
(new_commit, SubscriptionWorkerState::Idle),
] {
let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else {
panic!("expected one committed upsert");
};
assert_eq!(record.observation.as_ref().unwrap().worker.state, expected);
}
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
assert_eq!(
snapshot.take_snapshot()[0]
.observation
.as_ref()
.unwrap()
.worker
.state,
SubscriptionWorkerState::Idle
);
assert!(matches!(
subscription.try_recv(),
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
));
service.shutdown_async().await;
assert!(service.tasks.lock().unwrap().is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn snapshot_registration_is_atomic_with_concurrent_publication() {
let (store, service, _temp) = fixture().await;
+30 -8
View File
@@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError {
Replay,
#[error("source Worker is not a current member of this Workspace Runtime catalog")]
WorkerCatalogMembership,
#[error("source proof authority is unavailable")]
Unavailable,
#[error("source proof authority failed: {0}")]
Authority(String),
}
impl From<crate::Error> for WorkerMutationSourceProofError {
fn from(error: crate::Error) -> Self {
match error {
crate::Error::DbCapacityExceeded
| crate::Error::DbExecutorUnavailable
| crate::Error::DbOperationOutcomeUnknown
| crate::Error::Sqlite(_) => Self::Unavailable,
other => Self::Authority(other.to_string()),
}
}
}
pub async fn verify_worker_remove_source(
api: &WorkspaceApi,
proof: PresentedWorkerMutationSourceProof<'_>,
@@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with(
let trusted = store
.get_workspace_runtime_binding(&config.workspace_id, &unverified.iss)
.await
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?
.map_err(WorkerMutationSourceProofError::from)?
.filter(|record| record.revoked_at.is_none())
.ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?;
let public_key = trusted.public_key.as_str();
@@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with(
runtime_id: claims.iss.clone(),
worker_id: claims.worker_id.clone(),
};
let member = store
.get_worker_registry(&config.workspace_id, &worker)
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
let workspace_id = config.workspace_id.clone();
let catalog_store = Arc::clone(store);
store
.db_executor()
.run(move || {
let member = catalog_store
.get_worker_registry(&workspace_id, &worker)
.map_err(WorkerMutationSourceProofError::from)?;
if member.is_none() {
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
}
store
.require_current_worker_singleton_owner(&config.workspace_id, &worker)
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
catalog_store
.require_current_worker_singleton_owner(&workspace_id, &worker)
.map_err(WorkerMutationSourceProofError::from)?;
Ok(())
})
.await?;
let consumed_at = chrono::Utc::now().to_rfc3339();
let consumed = store
@@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with(
&consumed_at,
)
.await
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
.map_err(WorkerMutationSourceProofError::from)?;
if !consumed {
return Err(WorkerMutationSourceProofError::Replay);
}
@@ -1,3 +1,5 @@
use std::sync::Arc;
use worker_runtime::auth::{
RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims,
RuntimeRequestSourceExpectation, decode_runtime_request_source_claims,
@@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError {
pub(crate) exp: Option<i64>,
pub(crate) now: Option<i64>,
pub(crate) time_delta_seconds: Option<u64>,
/// Operational unavailability is not a proof or principal rejection.
pub(crate) availability: bool,
}
impl RuntimeRequestProofError {
pub fn is_availability_error(&self) -> bool {
self.availability
}
fn unavailable(mut self, reason: &'static str) -> Self {
self.reason = reason;
self.availability = true;
self
}
fn undecoded() -> Self {
Self {
reason: "invalid",
@@ -47,6 +61,7 @@ impl RuntimeRequestProofError {
exp: None,
now: None,
time_delta_seconds: None,
availability: false,
}
}
@@ -106,6 +121,18 @@ impl RuntimeRequestProofError {
}
}
impl From<crate::Error> for RuntimeRequestProofError {
fn from(error: crate::Error) -> Self {
let reason = match error {
crate::Error::DbCapacityExceeded => "db_capacity_exceeded",
crate::Error::DbExecutorUnavailable => "db_executor_unavailable",
crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown",
_ => "database_authority_unavailable",
};
Self::undecoded().unavailable(reason)
}
}
/// Domain-separated fixed-length fingerprints, not raw request-derived IDs.
/// Operators can correlate a known ID by applying this same function.
pub fn diagnostic_id_hash(id: &str) -> String {
@@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof(
body_digest: &str,
) -> Result<VerifiedRuntimeRequestSource, RuntimeRequestProofError> {
verify_runtime_request_source_proof_with_store(
api.store.as_ref(),
Arc::clone(&api.store),
&api.config,
proof,
workspace_id,
@@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof(
#[allow(clippy::too_many_arguments)]
pub async fn verify_runtime_request_source_proof_with_store(
store: &dyn ControlPlaneStore,
store: Arc<dyn ControlPlaneStore>,
config: &ServerConfig,
proof: &str,
workspace_id: &str,
@@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store(
#[allow(clippy::too_many_arguments)]
pub(crate) async fn verify_runtime_request_source_proof_with_clock(
store: &dyn ControlPlaneStore,
store: Arc<dyn ControlPlaneStore>,
config: &ServerConfig,
proof: &str,
workspace_id: &str,
@@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
let unverified = decode_runtime_request_source_claims(proof)
.map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?;
let diagnostic = RuntimeRequestProofError::decoded(&unverified);
let audience = remote_audience(config, workspace_id)
.map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?;
let audience = remote_audience(config, workspace_id).map_err(|_| {
diagnostic
.clone()
.unavailable("backend_audience_unavailable")
})?;
let trusted = store
.get_workspace_runtime_binding(workspace_id, &unverified.iss)
.await
.map_err(|_| {
diagnostic
.clone()
.reason("runtime_trust_authority_unavailable")
.unavailable("runtime_trust_authority_unavailable")
})?
.filter(|record| record.revoked_at.is_none())
.ok_or_else(|| {
@@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
&consumed_at,
)
.await
.map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))?
.map_err(|_| {
diagnostic
.clone()
.unavailable("replay_authority_unavailable")
})?
{
return Err(diagnostic.reason("replay"));
}
@@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
runtime_id: claims.iss.clone(),
worker_id: worker_id.to_owned(),
};
let workspace_id = workspace_id.to_owned();
let executor = store.db_executor();
let stage_diagnostic = diagnostic.clone();
// Only synchronous catalog authority lives in this closure. The trust
// and JTI async wrappers above already offload through the executor.
executor
.run(move || {
let member = store
.get_worker_registry(workspace_id, &worker)
.get_worker_registry(&workspace_id, &worker)
.map_err(|_| {
diagnostic
stage_diagnostic
.clone()
.reason("worker_catalog_authority_unavailable")
.unavailable("worker_catalog_authority_unavailable")
})?;
let reserved = store
.has_active_worker_create_reservation(workspace_id, &worker)
.has_active_worker_create_reservation(&workspace_id, &worker)
.map_err(|_| {
diagnostic
stage_diagnostic
.clone()
.reason("worker_reservation_authority_unavailable")
.unavailable("worker_reservation_authority_unavailable")
})?;
if member.is_none() && !reserved {
return Err(diagnostic.reason("worker_catalog_membership"));
return Err(stage_diagnostic.reason("worker_catalog_membership"));
}
store
.require_current_worker_singleton_owner(workspace_id, &worker)
.map_err(|_| {
diagnostic
.require_current_worker_singleton_owner(&workspace_id, &worker)
.map_err(|error| {
if matches!(
error,
crate::Error::DbCapacityExceeded
| crate::Error::DbExecutorUnavailable
| crate::Error::DbOperationOutcomeUnknown
| crate::Error::Sqlite(_)
) {
stage_diagnostic
.clone()
.unavailable("worker_singleton_authority_unavailable")
} else {
stage_diagnostic
.clone()
.reason("worker_singleton_authority_denied")
}
})?;
Ok(())
})
.await
.map_err(|error: RuntimeRequestProofError| {
if error.verification == "unverified" && error.is_availability_error() {
diagnostic.clone().unavailable(error.reason)
} else {
error
}
})?;
}
Ok(VerifiedRuntimeRequestSource {
@@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
authentication_binding_id: trusted.binding_id,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::{
AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode,
WorkspaceRuntimeBinding,
};
use std::time::Duration;
use worker_runtime::auth::{
BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner,
request_body_digest,
};
const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa";
const NOW: i64 = 1_700_000_000;
const PATH: &str = "/api/runtime/v1/resources/fetch";
// rusqlite's busy handler accepts a function pointer, not a closure. Each
// fixture owns a dedicated single-worker executor, so this thread-local
// handshake is isolated from all parallel tests and product connections.
thread_local! {
static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell<Option<(
tokio::sync::oneshot::Sender<()>,
std::sync::mpsc::Receiver<()>,
)>> = const { std::cell::RefCell::new(None) };
}
fn acknowledge_sqlite_contention(_attempt: i32) -> bool {
SQLITE_BUSY_HANDSHAKE.with(|slot| {
let Some((blocked, retry)) = slot.borrow_mut().take() else {
return false;
};
let _ = blocked.send(());
// The test releases SQLite's writer lock before allowing retry.
// Dropping the sender on panic also releases this worker promptly.
retry.recv_timeout(Duration::from_secs(3)).is_ok()
})
}
async fn fixture(
root: &std::path::Path,
) -> (Arc<SqliteWorkspaceStore>, ServerConfig, String, String) {
let db_path = root.join("auth.db");
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
workers: std::num::NonZeroUsize::new(1).unwrap(),
queued: std::num::NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let store = Arc::new(
SqliteWorkspaceStore::open_for_test(&db_path)
.unwrap()
.with_db_executor(executor),
);
store
.upsert_account(&AccountRecord {
account_id: "account-auth".into(),
kind: "user".into(),
handle: "auth".into(),
display_name: "Auth".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
let workspace = WorkspaceRecord {
workspace_id: WORKSPACE.into(),
owner_account_id: "account-auth".into(),
display_name: "Auth".into(),
state: "active".into(),
created_at: "1".into(),
updated_at: "1".into(),
};
store.upsert_workspace(&workspace).await.unwrap();
let config = ServerConfig {
workspace_id: WORKSPACE.into(),
workspace_display_name: "Auth".into(),
workspace_created_at: "1".into(),
workspace_execution_root: root.to_owned(),
database_path: db_path,
data_root: root.to_owned(),
embedded_runtime_store_root: root.join("runtime"),
static_assets_dir: None,
auth: crate::server::AuthConfig::Passkey {
rp_id: "localhost".into(),
origin: "http://localhost".into(),
public_base_url: "http://localhost".into(),
cookie_name: "session".into(),
},
max_records: 10,
repositories: vec![],
runtime_event_sources: vec![],
remote_runtime_sources: vec![],
backend_base_url: Some("server-auth".into()),
};
let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap();
store
.upsert_workspace_runtime_binding_record(
WorkspaceRuntimeBinding {
workspace_id: WORKSPACE.into(),
runtime_id: "runtime-auth".into(),
display_name: "Auth".into(),
base_url: "http://127.0.0.1:1".into(),
public_key: identity.public_key.clone(),
public_key_fingerprint: String::new(),
binding_id: String::new(),
authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer,
created_at: "1".into(),
updated_at: "1".into(),
revoked_at: None,
},
false,
)
.await
.unwrap();
let binding = ControlPlaneStore::get_workspace_runtime_binding(
store.as_ref(),
WORKSPACE,
"runtime-auth",
)
.await
.unwrap()
.unwrap();
let proof = RuntimeRequestSourceSigner::from_identity(&identity)
.issue(
"server-auth",
WORKSPACE,
None,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
b"{}",
NOW,
30,
)
.unwrap();
(store, config, proof, binding.binding_id)
}
#[tokio::test(flavor = "current_thread")]
async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() {
let root = tempfile::tempdir().unwrap();
let (store, config, proof, binding_id) = fixture(root.path()).await;
// A real independent SQLite writer prevents JTI insertion while trust
// reads remain possible. Rollback on Drop also releases this on panic.
let mut writer = rusqlite::Connection::open(&config.database_path).unwrap();
let lock = writer
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.unwrap();
let (blocked, contention_reached) = tokio::sync::oneshot::channel();
let (retry, retry_rx) = std::sync::mpsc::channel();
let setup_store = store.clone();
store
.db_executor()
.run(move || -> crate::Result<()> {
SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx)));
setup_store.with_conn(|conn| {
conn.busy_handler(Some(acknowledge_sqlite_contention))?;
Ok(())
})
})
.await
.unwrap();
let request_store = store.clone();
let request_config = config.clone();
let request_proof = proof.clone();
let task = tokio::spawn(async move {
verify_runtime_request_source_proof_with_clock(
request_store,
&request_config,
&request_proof,
WORKSPACE,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
&request_body_digest(b"{}"),
&|| NOW,
)
.await
});
tokio::time::timeout(Duration::from_secs(1), contention_reached)
.await
.expect(
"SQLite JTI persistence must report real writer contention without blocking Tokio",
)
.unwrap();
// The busy callback acknowledges an actual SQLite write blocked by the
// independent writer, then holds retry until explicitly released.
assert!(
!task.is_finished(),
"JTI must remain pending while SQLite writer is locked"
);
let control = tokio::spawn(async { "control-responsive" });
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), control)
.await
.unwrap()
.unwrap(),
"control-responsive"
);
// This timer is the behavior being tested, not a guessed wait for SQL.
tokio::time::timeout(
Duration::from_secs(1),
tokio::time::sleep(Duration::from_millis(1)),
)
.await
.unwrap();
assert!(
!task.is_finished(),
"timer must advance while JTI remains blocked"
);
lock.rollback().unwrap();
retry.send(()).unwrap();
let source = tokio::time::timeout(Duration::from_secs(3), task)
.await
.unwrap()
.unwrap()
.unwrap();
assert_eq!(source.runtime_id, "runtime-auth");
assert_eq!(source.authentication_binding_id, binding_id);
let replay = verify_runtime_request_source_proof_with_clock(
store,
&config,
&proof,
WORKSPACE,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
&request_body_digest(b"{}"),
&|| NOW,
)
.await
.unwrap_err();
assert_eq!(replay.reason, "replay");
assert!(!replay.is_availability_error());
}
#[tokio::test]
async fn saturated_database_is_availability_failure_not_invalid_proof() {
let root = tempfile::tempdir().unwrap();
let (store, config, proof, _) = fixture(root.path()).await;
// Fill the isolated executor with a blocked job and queued jobs. No
// scheduling sleeps: the worker explicitly acknowledges its start.
let executor = store.db_executor();
let (started, started_rx) = tokio::sync::oneshot::channel();
let (release, release_rx) = std::sync::mpsc::channel();
let blocker_executor = executor.clone();
let blocker = tokio::spawn(async move {
blocker_executor
.run(move || -> crate::Result<()> {
let _ = started.send(());
release_rx.recv().unwrap();
Ok(())
})
.await
});
tokio::time::timeout(Duration::from_secs(1), started_rx)
.await
.unwrap()
.unwrap();
let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) }));
assert!(futures::poll!(queued.as_mut()).is_pending());
let outcome = tokio::time::timeout(
Duration::from_secs(1),
verify_runtime_request_source_proof_with_clock(
store,
&config,
&proof,
WORKSPACE,
BACKEND_RESOURCE_FETCH_PERMISSION,
"POST",
PATH,
&request_body_digest(b"{}"),
&|| NOW,
),
)
.await;
// Release before assertions so a failed contract never strands a worker.
release.send(()).unwrap();
blocker.await.unwrap().unwrap();
queued.await.unwrap();
let error = outcome
.expect("capacity rejection must not wait for queued SQL")
.unwrap_err();
assert_eq!(error.reason, "runtime_trust_authority_unavailable");
assert!(error.is_availability_error());
}
}
@@ -285,7 +285,7 @@ mod tests {
#[tokio::test]
async fn create_is_atomic_and_exact_retries_converge() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
let service = WorkspaceCatalogService::new(
store.clone(),
Arc::new(
@@ -339,7 +339,7 @@ mod tests {
#[test]
fn create_rejects_noncanonical_repository_keys_without_persisting_a_workspace() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
let service = WorkspaceCatalogService::new(
store.clone(),
Arc::new(
@@ -378,7 +378,7 @@ mod tests {
for with_repository in [false, true] {
let temp = tempfile::tempdir().unwrap();
let database_path = temp.path().join("server.db");
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
let owner_account_id = owner_account(store.as_ref());
let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default());
let service = WorkspaceCatalogService::new(
@@ -418,7 +418,7 @@ mod tests {
drop(service);
drop(store);
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
let restarted = WorkspaceCatalogService::new(store.clone(), materials);
let created = restarted.create(request, owner_account_id).unwrap();
let identity = store
@@ -435,7 +435,7 @@ mod tests {
for with_repository in [false, true] {
let temp = tempfile::tempdir().unwrap();
let database_path = temp.path().join("server.db");
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
let owner_account_id = owner_account(store.as_ref());
let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default());
let service = WorkspaceCatalogService::new(store.clone(), materials.clone());
@@ -486,7 +486,7 @@ mod tests {
drop(service);
drop(store);
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
let restarted = WorkspaceCatalogService::new(store.clone(), materials);
let created = restarted.create(request, owner_account_id).unwrap();
let identity = store
@@ -500,7 +500,7 @@ mod tests {
#[tokio::test]
async fn idempotency_key_reuse_with_different_payload_is_rejected() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
let owner_account_id = owner_account(store.as_ref());
let service = WorkspaceCatalogService::new(
store,
@@ -532,7 +532,7 @@ mod tests {
#[test]
fn repository_free_create_initializes_authority_and_concurrent_retries_converge() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
let owner = owner_account(store.as_ref());
let service = WorkspaceCatalogService::new(
store.clone(),
@@ -611,7 +611,7 @@ mod tests {
#[test]
fn replay_reads_current_repository_at_initial_key_and_never_recreates_it() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
let owner = owner_account(store.as_ref());
let service = WorkspaceCatalogService::new(
store.clone(),
@@ -673,13 +673,13 @@ mod tests {
repository: None,
};
let (owner, created) = {
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
let owner = owner_account(store.as_ref());
let service = WorkspaceCatalogService::new(store, materials.clone());
let created = service.create(request.clone(), owner.clone()).unwrap();
(owner, created)
};
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
let reopened = WorkspaceCatalogService::new(store.clone(), materials);
let replay = reopened.create(request, owner).unwrap();
assert_eq!(replay.workspace, created.workspace);
@@ -739,7 +739,7 @@ mod tests {
#[test]
fn create_rejects_non_user_account_owners() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
store
.upsert_account(&AccountRecord {
account_id: "organization-owner".to_string(),
@@ -777,7 +777,7 @@ mod tests {
#[test]
fn catalog_list_is_scoped_to_the_required_owner_account() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
let owner_a = owner_account(store.as_ref());
let owner_b = "account-owner-b".to_string();
store
@@ -845,7 +845,7 @@ mod tests {
#[test]
fn remote_repository_creation_persists_typed_source_without_auth_metadata() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
let owner_account_id = owner_account(store.as_ref());
let service = WorkspaceCatalogService::new(
store.clone(),
@@ -236,7 +236,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
"Workspace deletion request exceeds bounded field limits".to_string(),
));
}
self.with_transaction(|tx| {
self.with_immediate_transaction(|tx| {
if let Some(existing) = read_operation(tx, &request.operation_id)? {
if existing.actor_account_id != actor_account_id {
return Err(Error::WorkspacePermissionDenied(
@@ -369,7 +369,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
child_operation_id: &str,
) -> Result<WorkspaceDeletionOperationResponse> {
validate_operation_id(child_operation_id)?;
self.with_transaction(|tx| {
self.with_immediate_transaction(|tx| {
let operation = read_operation(tx, operation_id)?
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?
.response;
@@ -411,7 +411,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
failure_category: Option<&str>,
) -> Result<WorkspaceDeletionOperationResponse> {
validate_operation_projection(child_operation_ids, blockers)?;
self.with_transaction(|tx| {
self.with_immediate_transaction(|tx| {
let current = read_operation(tx, operation_id)?
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?
.response;
@@ -462,7 +462,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
&self,
operation_id: &str,
) -> Result<WorkspaceDeletionOperationResponse> {
self.with_transaction(|tx| {
self.with_immediate_transaction(|tx| {
tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?;
let operation = read_operation(tx, operation_id)?
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?;
@@ -922,7 +922,7 @@ mod tests {
use tempfile::tempdir;
fn setup() -> (SqliteWorkspaceStore, String, String) {
seed_workspaces(SqliteWorkspaceStore::in_memory().expect("store"))
seed_workspaces(SqliteWorkspaceStore::in_memory_for_test().expect("store"))
}
fn seed_workspaces(store: SqliteWorkspaceStore) -> (SqliteWorkspaceStore, String, String) {
@@ -989,7 +989,7 @@ mod tests {
let dir = tempdir().expect("tempdir");
let path = dir.path().join("server.db");
let (store, owner, workspace_id) =
seed_workspaces(SqliteWorkspaceStore::open(&path).expect("store"));
seed_workspaces(SqliteWorkspaceStore::open_for_test(&path).expect("store"));
for workspace in ["workspace-a", "workspace-b"] {
let backend = ticket::SqliteTicketBackend::open_verified(&path, workspace).unwrap();
let initial = backend
@@ -1019,7 +1019,7 @@ mod tests {
// Seed domain handling evidence under a transaction. This fixture
// exercises owner purge, not singleton authentication or delivery.
store
.with_transaction(|tx| {
.with_immediate_transaction(|tx| {
ticket::sqlite_update_ticket_assignment_tracking_in_transaction(
tx,
workspace,
@@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost {
impl WorkspaceDriveHost {
pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result<Self> {
let storage = storage.clone();
let registration = tokio::task::spawn_blocking(move || Drive::register(&storage))
.await
.map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))?
.map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?;
let registration = crate::db_executor::DbExecutor::shared()
.run(move || {
Drive::register(&storage).map_err(|error| {
Error::Store(format!("failed to register Drive storage: {error}"))
})
})
.await?;
Ok(Self {
registration,
purged: AtomicBool::new(false),
@@ -80,7 +83,8 @@ impl WorkspaceDriveHost {
}
let storage = storage.clone();
let registration = self.registration.clone();
let drive = tokio::task::spawn_blocking(move || {
let drive = crate::db_executor::DbExecutor::shared()
.run(move || {
reject_symlink_ancestors(&root)?;
// Domain IMMEDIATE transactions lock the attached server authority
// as well as Drive metadata. Deletion reservation and cached-handle
@@ -93,8 +97,7 @@ impl WorkspaceDriveHost {
)
.map_err(Error::from)
})
.await
.map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??;
.await?;
*cached = Some(drive.clone());
Ok(drive)
}
@@ -109,9 +112,9 @@ impl WorkspaceDriveHost {
return Ok(());
}
let drive = self.open(storage, root, authority_database).await?;
tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from))
crate::db_executor::DbExecutor::shared()
.run(move || drive.fence().map_err(Error::from))
.await
.map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))?
}
pub(crate) async fn purge(
@@ -124,7 +127,8 @@ impl WorkspaceDriveHost {
return Ok(());
}
let drive = self.open(storage, root.clone(), authority_database).await?;
tokio::task::spawn_blocking(move || -> Result<()> {
crate::db_executor::DbExecutor::shared()
.run(move || -> Result<()> {
reject_symlink_ancestors(&root)?;
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
drive.purge()?;
@@ -136,8 +140,7 @@ impl WorkspaceDriveHost {
Err(error) => Err(error.into()),
}
})
.await
.map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??;
.await?;
// A later signing-material/finalization failure can retry after the
// feature DB has been closed and removed by FeatureStorage.delete.
self.purged.store(true, Ordering::Release);
@@ -780,7 +780,7 @@ mod tests {
let temp = tempfile::tempdir().unwrap();
let database_path = temp.path().join("server.db");
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
store
.upsert_account(&AccountRecord {
account_id: "account-1".to_string(),
@@ -833,7 +833,7 @@ mod tests {
drop(failing_service);
drop(store);
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
let service = WorkspaceSigningIdentityService::new(store.clone(), materials.clone());
let provisioned = service
.provision_existing("workspace-1", "account-1")
@@ -962,7 +962,7 @@ mod tests {
);
drop(service);
drop(store);
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
let restarted = WorkspaceSigningIdentityService::new(store.clone(), materials.clone());
let recovered = restarted
.provision_existing("workspace-2", "account-1")
@@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription<S>(
worker_id,
runtime_id: Some(runtime_id),
} => {
let connection = match WorkspaceWorker::resolve(
let connection = match WorkspaceWorker::resolve_async(
&api,
&runtime_id,
worker_id.as_str(),
) {
)
.await
{
Ok(worker) => worker
.connect_protocol(&context)
.await
@@ -366,7 +368,8 @@ async fn run_workspace_workers(
) {
let Ok(mut projection) = api
.worker_projection
.subscribe_ordered(api.config.max_records.max(1))
.subscribe_ordered_async(api.config.max_records.max(1))
.await
else {
let _ = send_rejected(
&outbound,
@@ -377,11 +380,22 @@ async fn run_workspace_workers(
.await;
return;
};
let mut workers = projection
.take_snapshot()
.into_iter()
.filter_map(|record| project_registry_worker(&api, record))
.collect::<Vec<_>>();
let Ok(mut workers) = project_registry_workers_async(
api.store.clone(),
api.config.workspace_id.clone(),
projection.take_snapshot(),
)
.await
else {
let _ = send_rejected(
&outbound,
request_id,
SubscriptionRejectionCode::Internal,
"failed to enrich durable Worker projection".to_string(),
)
.await;
return;
};
sort_workers(&mut workers);
if send_frame(
&outbound,
@@ -406,7 +420,29 @@ async fn run_workspace_workers(
for change in event.changes {
let payload = match change {
crate::worker_projection::WorkerProjectionChange::Upsert(record) => {
let Some(worker) = project_registry_worker(&api, record) else {
let workers = project_registry_workers_async(
api.store.clone(),
api.config.workspace_id.clone(),
vec![record],
)
.await;
let Ok(mut workers) = workers else {
// Admission failure cannot silently lose a live
// update: terminate and require a fresh snapshot.
let _ = send_frame(
&outbound,
SubscriptionFrame::new(SubscriptionFramePayload::Event(
SubscriptionEvent::SubscriptionClosed {
subscription_id: subscription_id.clone(),
code: SubscriptionTerminationCode::Lagged,
message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(),
},
)),
)
.await;
return;
};
let Some(worker) = workers.pop() else {
continue;
};
SubscriptionEventPayload::WorkerUpserted { worker }
@@ -450,8 +486,28 @@ async fn run_workspace_workers(
}
}
async fn project_registry_workers_async(
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
workspace_id: String,
records: Vec<crate::store::WorkerRegistryProjectionRecord>,
) -> crate::Result<Vec<SubscriptionWorker>> {
store
.db_executor()
.run(move || {
// Keep the committed Worker payload; only the existing Workspace-scoped
// workdir/history/repository enrichment reads occur inside this job.
Ok(records
.into_iter()
.filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record))
.collect())
})
.await
}
// Synchronous helpers stay synchronous; they run only inside the job above.
fn project_registry_worker(
api: &WorkspaceApi,
store: &dyn crate::store::ControlPlaneStore,
workspace_id: &str,
record: crate::store::WorkerRegistryProjectionRecord,
) -> Option<SubscriptionWorker> {
let runtime_id = record.registry.worker.runtime_id.clone();
@@ -491,29 +547,25 @@ fn project_registry_worker(
attempt_id: job.attempt_id,
purpose: job.purpose,
});
if !project_working_directory(api, &record.registry.worker, &mut worker) {
if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) {
return None;
}
Some(worker)
}
fn project_working_directory(
api: &WorkspaceApi,
store: &dyn crate::store::ControlPlaneStore,
workspace_id: &str,
worker_ref: &RuntimeWorkerRef,
worker: &mut SubscriptionWorker,
) -> bool {
let Ok(links) = api
.store
.list_worker_workdir_links(&api.config.workspace_id, worker_ref)
else {
let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else {
return false;
};
if !links.is_empty() {
let mut attachments = Vec::with_capacity(links.len());
for link in links {
let Ok(Some(workdir)) = api
.store
.get_workdir_registry(&api.config.workspace_id, &link.workdir_id)
let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id)
else {
return false;
};
@@ -533,9 +585,8 @@ fn project_working_directory(
}
worker.workdir_attachments = attachments;
} else {
let Ok(has_link_history) = api
.store
.worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref)
let Ok(has_link_history) =
store.worker_workdir_link_history_exists(workspace_id, worker_ref)
else {
return false;
};
@@ -543,13 +594,17 @@ fn project_working_directory(
worker.workdir_attachments.clear();
}
}
project_repository_keys(api, worker)
project_repository_keys(store, workspace_id, worker)
}
fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool {
fn project_repository_keys(
store: &dyn crate::store::ControlPlaneStore,
workspace_id: &str,
worker: &mut SubscriptionWorker,
) -> bool {
project_repository_keys_with(worker, |repository_id| {
api.store
.get_repository(&api.config.workspace_id, repository_id)
store
.get_repository(workspace_id, repository_id)
.ok()
.flatten()
.map(|repository| repository.repository_key)
@@ -634,9 +689,8 @@ mod tests {
protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap()
}
#[test]
fn external_workdir_attachment_does_not_remove_workspace_worker() {
let mut worker = SubscriptionWorker {
fn external_worker() -> SubscriptionWorker {
SubscriptionWorker {
restore_observation_token: None,
worker_id: worker_id(),
runtime_id: Some("runtime-1".to_string()),
@@ -660,8 +714,12 @@ mod tests {
.unwrap(),
},
],
};
}
}
#[test]
fn external_workdir_attachment_does_not_remove_workspace_worker() {
let mut worker = external_worker();
assert!(project_repository_keys_with(&mut worker, |_| {
panic!("External Workdir must not require Repository projection")
}));
@@ -673,6 +731,110 @@ mod tests {
);
}
#[tokio::test(flavor = "current_thread")]
async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository()
{
use crate::store::{
AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord,
WorkspaceRecord,
};
let store = std::sync::Arc::new(
SqliteWorkspaceStore::in_memory_for_test()
.unwrap()
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
);
store
.upsert_account(&AccountRecord {
account_id: "owner".into(),
kind: "user".into(),
handle: "owner".into(),
display_name: "Owner".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: "workspace-1".into(),
owner_account_id: "owner".into(),
display_name: "Enrichment".into(),
state: "active".into(),
created_at: "1".into(),
updated_at: "1".into(),
})
.await
.unwrap();
let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1");
store
.upsert_worker_registry(&WorkerRegistryRecord {
workspace_id: "workspace-1".into(),
worker: worker_ref.clone(),
display_name: "Durable name".into(),
profile: Some("builtin:coder".into()),
retention_state: "normal".into(),
transcript_ref: None,
session_ref: None,
summary_ref: None,
diagnostics_ref: None,
created_at: "1".into(),
updated_at: "1".into(),
})
.unwrap();
store
.apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2")
.unwrap();
let committed = store
.worker_registry_projection("workspace-1", &worker_ref)
.unwrap()
.unwrap();
// Change authority after capturing the payload, as can happen while async
// enrichment is queued. It must not re-query and replace the committed Worker.
let mut later = external_worker();
later.state = protocol::subscription::SubscriptionWorkerState::Running;
store
.apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3")
.unwrap();
let projected = project_registry_workers_async(
store.clone(),
"workspace-1".into(),
vec![committed.clone()],
)
.await
.unwrap();
assert_eq!(projected.len(), 1);
let worker = &projected[0];
assert_eq!(
worker.state,
protocol::subscription::SubscriptionWorkerState::Idle
);
assert_eq!(
worker.availability,
protocol::subscription::SubscriptionWorkerAvailability::Observed
);
assert_eq!(worker.display_name.as_deref(), Some("Durable name"));
assert_eq!(worker.profile.as_deref(), Some("builtin:coder"));
assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1"));
assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1"));
assert_eq!(
worker.workdir_attachments,
external_worker().workdir_attachments
);
let mut unresolved = committed;
unresolved
.observation
.as_mut()
.unwrap()
.worker
.workdir_attachments[0]
.repository_id = Some("missing-repository".into());
assert!(
project_registry_workers_async(store, "workspace-1".into(), vec![unresolved])
.await
.unwrap()
.is_empty()
);
}
async fn run_with_events(events: mpsc::Receiver<protocol::Event>) -> SubscriptionResponse {
let (control_outbound, mut control_receiver) = mpsc::channel(1);
let (protocol_outbound, _protocol_receiver) = mpsc::channel(1);
+115
View File
@@ -0,0 +1,115 @@
# Backend bounded synchronous execution
## Ownership and admission
`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner.
Use `executor.run(move || synchronous_operation(...)).await` with owned input and
services. The closure owns the **whole** synchronous operation, including connection
acquisition, mutex wait, the existing SQL transaction, commit, and ordered local
publication. Do not split source authorization, receipts, CAS, audit, or saved
results around an await. Do not change connection ownership or introduce a read pool.
Production stores share one DB worker and a queue of 32 jobs. Admission uses a
nonblocking queue send and a separate actual-job budget (worker count plus queue
capacity); rejection means the operation was **not accepted**. HTTP maps capacity
and executor unavailability to 503, not authentication failure or deletion conflict.
Tests can inject a dedicated executor with explicit limits, without process-global
settings or environment variables. Ordinary SQLite fixtures use `open_for_test`,
`in_memory_for_test`, or `from_connection_for_test`: their clones share their own
1-worker/32-queued owner, not the production singleton or another fixture's drain.
Server/Workspace fixture constructors likewise inject a lazy `HostExecutors` owner;
all child Workspaces and weak retention adapters use that same owner. Host and
cleanup still have the production 2-worker/16-queued limits. Dropping the final
sender closes its queue after accepted work finishes; idle fixture threads are
not kept in a process-global fixture cache.
The restore intent journal and Workspace deletion use
`with_immediate_transaction`: read authority/journal state only after acquiring
SQLite write ownership. A DEFERRED read-to-write upgrade can return BUSY without
invoking SQLite's busy handler when another connection is writing. Moving these
whole helpers to an offloaded caller must not create an automatic mutation retry.
Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter:
- does not interrupt, resubmit, or retry SQL;
- does not return its queue/job slot before execution and discarded-result cleanup;
- does not release the request lifecycle lease before actual completion.
Synchronous post-commit continuations inherit the admitted lifecycle source when
submitting another bounded job. This is not implicit inheritance by detached Tokio
tasks. An operation panic has an unknown outcome even if SQL already committed;
return a structured unknown-outcome error and diagnostic, never retry the mutation.
A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all
jobs accepted before its barrier is first polled and waits for actual cleanup; it
is not a queue close and does not include later admissions. Lifecycle callers must
stop their producers/admission before destroying resources.
## Runtime/network boundaries
A synchronous Runtime workflow must not monopolize the sole DB worker while
waiting for callbacks that themselves need DB authority. The Server's separate
finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job
ownership mechanism, not an unbounded `spawn_blocking` queue.
Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can
use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an
owned workflow needs async resource locks, `run_host_workflow` polls it on a Host
worker using the original Tokio handle; do not build a replacement runtime for
cached services and do not submit to and await the Host lane from inside itself.
Control/session guards needed by an accepted mutation belong to that actual job.
Backend Job cleanup has a third finite lane (two workers, 16 queued jobs), so a
long Run teardown wait cannot starve ordinary Host control/stop requests. It is
admitted before its durable cleanup claim. Rejection leaves Pending/Failed
ownership recoverable instead of abandoning an Executing claim. Ticket
checker dispatch and timeout processing use bounded admission; overload leaves the
saved reservation/deadline diagnosable. They are not automatically retried after an
unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof
of model processing.
Remote subscription and Workdir authorization are async callbacks: their complete
live binding check and fresh request signing are offloaded together. A queue error
must not fall back to a configured bearer token. Synchronous CLI/migration callers
and existing synchronous Runtime adapters retain their explicit blocking boundary.
## Caller recipe (including assignment APIs)
```rust
let store = api.store.clone();
let workspace_id = api.workspace_id().to_owned();
let worker = worker.clone();
let record = api.store.db_executor().run(move || {
store.get_worker_registry(&workspace_id, &worker)
}).await?;
```
This is an illustration, not a new authority API. Assignment delivery scan,
reserve, prepare, acknowledgement and start authorization must keep their existing
whole transaction helpers. Additional assignment diagnostics, answer mutations,
and Ticket queries use the same owned caller boundary; do not add a second ledger
or reconstruct processing evidence from Worker liveness.
## Regression evidence
- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread
timer/control task progresses; accepted/cancelled queue bounds; rollback; panic
before/after commit; discarded reply cleanup and retained authority; child jobs.
- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope
opening/mutation under a separate SQLite connection's write lock; overload;
cancelled HTTP jobs and Workspace deletion drain; same-budget fixture isolation,
saturated/cancelled Host jobs versus cleanup and paused Tokio time; restore intent
contention before the first authority read, with one persisted tuple and no retry.
The terminal checker fixture keeps its original Tokio driver alive through accepted cleanup,
then shuts down projection producers and drains actual DB jobs.
- `worker_projection::tests`: real SQL contention, commit/payload publication,
ordered snapshot registration, and replacement draining a cancelled consumer's
accepted SQL before applying a new Runtime snapshot.
- Auth/proof and Runtime subscription tests protect availability classification,
replay rejection, and no bearer fallback.
- `workdir`'s `http-client` tests protect awaited fresh authorization and exact
request claims before any HTTP send.
These fixtures use temporary authority data and observable synchronization, not
live databases or a fixed sleep to guess completion. This change does not claim a
latency benchmark or a measured production incident.
@@ -0,0 +1,166 @@
# T-748: default-parallel suite failure diagnosis
## Sources and preserved observations
- Original local candidate: `61533fadb6b39465af60cb55aed2b4d0650319cd`.
- Comparison source: `56795fb92cb9def9bcd701bd51d92e99526f8eff` (develop/T742).
- Same implementation branch: `work/T-748-bounded-db-execution`.
- The original `target/t748-server-tests.log` remains untouched. Its 97 FAILED
lines and 600-second timeout were symptoms, not proof of one cause.
- Initial diagnosis used `cargo test -p yoi-workspace-server -- --nocapture`,
default libtest parallelism, without changing queue limits, features, or startup
settings. `target/t748-diagnostic-environment.txt` records the environment;
this host reports 24 available CPUs. Diagnostic captures were bounded to
90/150 seconds to obtain errors before the original timeout, not counted as
successful full-suite runs.
Logs under `target/` are local diagnostic artifacts, not tracked review inputs.
This report records their meaning and the permanent regressions that reproduce it.
## 1. Different fixtures consumed one global Host admission budget
The first non-intentional errors in `target/t748-server-nocapture.log` were
`DbCapacityExceeded` in manual Worker setup (`server.rs` at the then-current
line 50614), followed by assignment fixture construction (52990/53045) and
HTTP 503 responses. The deliberate before/after-commit executor panic tests
are different: those passed and verified unknown-outcome diagnostics.
Temporary submit/start/cleanup instrumentation recorded submitting test names,
closure types, actual job IDs and the execution-owner identity. It showed a
single Host lane with `max=18`, with accepted jobs from unrelated fixtures.
A rejection snapshot contained `{32,45,46,...,61}`. The complete diagnostic
capture later contained cleanup for every submitted Host job on that owner;
it did not show permanently abandoned accepted Host jobs. Instrumentation was
removed before the source was committed/published.
`target/t748-live-process-18.txt` sampled the test process and threads at the
actual rejection. That sample had 126 threads; the Host workers were runnable,
and the process was using approximately 17 CPU cores. There was no captured
thread-creation EAGAIN/unavailable error. This evidence does not establish an
OS thread/CPU hard limit as the cause. No test processes remained after the
bounded diagnostic run or the completed suite.
A representative failure passed alone (0.56 seconds); the eight logging cases
passed with four test threads (2.46 seconds). The isolated comparison source's
full default-parallel suite passed: 931 library tests, one pre-existing ignored
migration fixture, 11 binary tests; library duration 168.74 seconds. This did
not prove that all new failures were capacity-related, but did distinguish the
new shared-admission rejection from an observed baseline failure.
### Ownership fix (`f3d4aba5`)
Production budgets remain shared and finite: DB 1 worker/32 queued, Host 2/16,
cleanup 2/16. Ordinary fixtures now explicitly own a finite DB executor and lazy
Host/cleanup owners at those same limits. API clones, child Workspace APIs, and
weak retention adapters use their injected owner. No global fixture lane cache,
parallelism reduction, unlimited queue, or capacity-wait retry was added.
The permanent saturated-Host regression fills both running slots and all 16
queued slots, cancels waiters, advances paused Tokio time, and still observes
rejection until actual completion. Another fixture and the separate cleanup lane
can progress; draining restores the original budget and completes exactly 18
accepted jobs. Existing real-SQLite cancellation/lease/drain tests remain active.
## 2. SQL offload invalidated fixture assumptions about first poll and publication
After Host isolation the default-parallel suite finished in 171.57 seconds:
954 passed, three failed, one existing ignored. Actual errors were:
- removal subscriber saw a legitimate queued Runtime stop Upsert before Removed;
- external session-open expiry exceeded its one-second bound while its DB lane
was still shared across other fixtures;
- grant replacement returned success because one initial poll had only submitted
the offloaded authority read, not reached the held original-grant lock.
The ordinary DB fixtures were isolated too. The one-second expiry assertion was
not increased. Grant testing now drains its actual authority read then polls the
saved response to reach the held lock before replacing the grant. Removal testing
observes the committed stopped projection before taking the removal baseline.
The Removed payload assertion is unchanged. These are observable synchronization,
not sleeps or retries. Focused expiry, grant, and removal tests all passed.
A captured cleanup job also attempted to poll a Tokio timer after its fixture
runtime had shut down. The terminal checker fixture now retains that original
runtime through accepted Host and cleanup jobs, then stops projection producers
and drains actual DB work before ending. Terminal SQL state is not treated as
actual cleanup completion. The subsequent full run contained only the three
intentional executor panic diagnostics, not the shutdown-context panic.
The synchronization/teardown changes are in `df9d056d`.
## 3. Restore intent's DEFERRED read-to-write upgrade returned SQLite BUSY
The next default-parallel run finished in 171.48 seconds with 956 passed, two
failed, one existing ignored. Both failures were SQLite `DatabaseBusy`, extended
code 5, not capacity rejection:
- production-profile zero-input Orchestrator restore;
- pending manual Worker claim restore reconciliation (500 instead of 409).
Both failed alone too. `target/t748-busy-origin.log` captured the error and a
backtrace through `pin_internal_worker_restore_intent` -> the transaction helper
-> internal restore -> manual claim. The journal read authority first in a
DEFERRED transaction while the offloaded projection's other SQLite connection
could write. Such an upgrade can return BUSY without invoking SQLite's busy
handler; treating this as a retryable whole mutation would be incorrect.
A new real-SQLite regression holds an IMMEDIATE writer on another connection,
registers an owned per-connection busy observer, and attempts one journal pin.
The old helper fails deterministically with BUSY/5 before waiting for ownership
(`target/t748-deferred-upgrade-red.log`, 0.23 seconds). The fixed helper waits
before its first authority read, while a current-thread timer and control task
progress. After releasing the writer it persists exactly one requested tuple;
there is no application-level mutation retry.
`f597f1b5` makes this existing mutation helper explicitly
`with_immediate_transaction`. Restore journal and deletion authority reads,
mutation and commit remain in one transaction. Focused DB/HTTP tests (8), the
original restore and claim cases, and deletion tests (13) passed.
## 4. Why a pre-dispatch failure could become the 600-second suite timeout
The original long-running logging case joined an HTTP request with a fake provider
waiting for command dispatch. On pre-dispatch rejection the HTTP future can finish
with 503 while the receiver remains Pending: its channel is still owned by the
fixture, but no command will arrive. An unbounded `join!` waits forever in that
state and hides the actual error behind the outer suite timeout.
The permanent `saturated_host_before_command_start_returns_availability_without_a_provider_operation`
regression uses the real signed HTTP route with a controlled full Host budget. It
observes the final 503/`db_capacity_exceeded` response and the still-live Pending
provider receiver, with no process or command dispatch. The original logging
fixture now asserts an early HTTP response before waiting for both branches, so
an unexpected rejection reports its body immediately rather than hanging. This
reproduces the causal timeout mechanism; the old log itself has no stack trace
from the hung provider future.
The nine logging cases pass together at default parallelism (`49bfbfb2`).
## Artifact/source isolation during comparison
Initially the archived baseline used the existing Cargo target directory. That
replaced a same-package artifact/fingerprint; one later post-fix representative
command ran the baseline's 932-test binary (and `db_executor --list` had no tests).
That result was rejected as candidate evidence. All later candidate checks/tests
use `--target-dir target/t748-candidate-build`, separate from the baseline cache.
This changes artifact location, not test parallelism, features or admission policy.
The fresh candidate build listed the expected executor tests and total test count.
## Validation and remaining composition boundary
Final code source `49bfbfb2` passed the full default-parallel Server suite,
including the command-overload case: 960 passed, zero failures, one existing
ignored migration fixture; 11 binary tests, library duration 175.29 seconds.
The only captured panics were the three deliberate unknown-outcome tests.
Root `cargo check` compile closure passed (including TUI/Server), as did Workdir
default (153 tests) and `http-client` (156 tests) full suites, format/diff checks
and state-contract names (558 matches, zero unapproved, zero stale exceptions).
These commands used the isolated candidate artifact directory; libtest default
parallelism and the production budgets were unchanged. The MR records the exact
published source, which also includes this report and the boundary document.
No latency benchmark, production incident measurement, target integration,
Ticket conclusion, or running-environment update is claimed. T743 has now
published its caller inventory/source; its owner will compose the T748 facade and
new callers, then obtain review of that exact combined source. T748 does not copy
those domain APIs or wait on the former unpublished branch.
+6
View File
@@ -2,6 +2,12 @@
## Source identity and composition
This is the retained pre-composition caller inventory published with T-743
`9b0d42fdbc2920736c3418bb894f26140e962bbc`. Its descriptions of direct callers
below identify the work to move, not the current execution placement. The later
normal merge, actual DB/Host placement, ten new regressions, full-suite evidence
and unresolved upstream review findings are in `T-743-T748-composition.md`.
This inventory belongs to the T-743 commit carrying this file on `work/T-743-ticket-public-surfaces` (T-742 base `56795fb92cb9def9bcd701bd51d92e99526f8eff`). Its exact published SHA and MR ID are recorded in the Ticket publication handoff. It is not a copy of the unpublished T-748 adapter and is not evidence that async DB execution has been integrated or reviewed.
Normal-merge the validated public T-748 source into this existing work branch. Keep the MR's source/target selectors. Recheck the concrete new call graph below against the merged implementation; no placeholder executor, second connection authority, transaction split, fresh integration branch or force push is needed.
+130
View File
@@ -0,0 +1,130 @@
# T-743 / T-748 composition: execution-boundary evidence
## Source and status
T-743 continues on `work/T-743-ticket-public-surfaces`, MR
`01a12d19-b61b-79a2-baca-89241fd3c3d3` → `develop`. The published T-748
source `66bb9d0b825d72e20923003b273b787007f3aed8` was normally fetched and
merged into that same source branch, without conflicts, replacement selectors,
force push, or a copied execution facade. The composition commit carrying this
report also connects the added callers below. Its exact published hash is in the
Ticket handoff and provider-resolved MR source.
**This is a validated intermediate composition, not final independent approval.**
T-748 MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq3 requests changes on this
upstream source. Its two major findings are not fixed by the new assignment
caller work: `T748-DB-NETWORK-LANE` (Subjektiv mixed Runtime/DB placement) and
`T748-ASYNC-PROTOCOL-SIGNING` (cold protocol observation source signing).
That authoritative verdict was checked again after the combined full suite;
no newer published upstream source or verdict was present. The T-748 owner was
informed through its Ticket of the composition and requested to provide the
normally advanced validated fix source. Do not interpret a green suite as proof
that those untested boundaries are safe. Normal-merge that fix into this same
T-743 source, revalidate, and independently review the exact combined source.
## Caller placement
- Canonical and Browser assignment pages and sanitized diagnostics share one
implementation. Workspace scope, Ticket reference resolution and each whole
synchronous Store read run on the shared bounded DB owner. None is left as a
synchronous pre-call on a Tokio request thread.
- Typed Ticket query, including principal/handling/current-closure filtering
before pagination, retains T-748's whole authority/read/enrichment DB job.
The regression holds the **authority's actual candidate connection**, not the
separate API Store connection used by only some enrichment paths. This also
proves the empty-result path and wrong-principal filters offload correctly.
- Account assignment answers derive attribution from Backend-authenticated
context inside the owned DB closure. Runtime answers preserve the live current
Orchestrator discovery check on the finite Host lane, because Runtime worker
lookup can perform remote HTTP. The DB closure derives Runtime attribution
without I/O and the unchanged answer helper validates current singleton and
live binding **again under its IMMEDIATE write transaction**, including replay.
Target principal and request-body prose never supply author authority.
- The same answer helper keeps source authority, actionable-assignment gate,
assignment/event CAS, full operation fingerprint/replay, body, audit, receipt
and commit together. There is no await within that transaction and no automatic
mutation retry.
- The accepted SQL job admits the existing finite Host attention continuation
**after commit and before returning its result**. It inherits the request
lifecycle source, owns lookup/Runtime dispatch independently of the waiter and
does not wait on Runtime/network on the sole DB worker. A live HTTP waiter
observes best-effort notification completion; losing the waiter does not
cancel accepted SQL or accepted publication. Rejected Host publication retains
the existing saved-input scan/recovery and diagnostic behavior, not rollback
or a synthetic successful delivery.
- Normal Tools/WIP still converge on the canonical HTTP contracts. No second
ledger, query adapter, source authority, read pool or delivery implementation
was added. Existing T-748 set/clear/start/delivery boundaries remain intact.
## New regression proof
`crates/workspace-server/src/server_assignment_db_execution_tests.rs` contains
10 tests on fixture-owned bounded DB/Host owners, temporary databases and explicit
observable gates. Parent inspected the child's committed tool output, the actual
new test code and then the combined default-parallel suite, rather than treating
an end notification as proof. The child's delegated file scope was released.
The regressions prove:
1. Assignment/diagnostic read and Ticket reference resolution wait off the
current-thread executor on their actual Store connection mutex.
2. Typed principal/handling/closure query matrices, including empty results,
hold the candidate connection and filter before the page limit.
3. A real second SQLite connection's IMMEDIATE writer reaches the answer's owned
busy observer while the same Tokio runtime's timer and independent control
progress; one transaction preserves the answer/event/audit/receipt.
4. Saturated DB admission returns HTTP 503 and does not accept an answer.
5. Cancelling an accepted answer waiter retains its actual job slot/lifecycle
through SQL and Host publication; body/audit/receipt persist, dispatch is once,
and actual cleanup releases all observed leases.
6. Successful HTTP answer waits for its admitted publication without occupying
the DB worker; an independent DB callback can progress during publication.
7. Account and Browser receipt replay, stale CAS, operation mismatch and bad
cursor/error mapping retain their domain meanings.
8. Real signed non-Orchestrator source and signed proof replay are denied with
no answer/receipt; signed current Orchestrator canonical/Browser replay shares
one source-bound receipt, and another source cannot replay it.
9. Saturated **2-worker/16-queued Host** live-discovery rejects canonical/Browser
Runtime answers before SQL admission, without receipt/audit/publication or
retained lifecycle, while independent DB callbacks progress. This last test
injects authenticated context at generated handlers to isolate discovery
admission; the preceding tests exercise actual signed-proof middleware.
Read/query contention intentionally uses the actual connection mutex; answer
contention uses a separate real SQLite writer. Neither is described as a full
remote Runtime latency benchmark. No fixed completion sleeps, mutation retry,
parallelism reduction, process-global fixture budgets or new ignored tests were
introduced. Fixtures drain accepted Host/cleanup jobs, shut down projection
producers, then drain DB jobs before ending their original Tokio driver. Owned
busy-handler userdata and SQLite locks unregister/release through RAII.
## Validation on this composition
- Root `cargo check`: passed, normal default-member compile closure.
- `cargo test -p yoi-workspace-server --lib assignment_db_execution_tests`:
**10 passed, 0 failed**, 2.63s (child committed output). Parent reran the first
nine after separating Runtime discovery and included all ten in the full suite.
- `cargo test -p yoi-workspace-server`: original default parallelism,
**985 library passed, 0 failed, 1 pre-existing ignored**, 176.60s;
**11 binary passed**, doc tests passed. Log:
`target/t743-combined-server-suite.log`.
- `cargo test -p workdir`: **153 passed** across library/integration targets;
`cargo test -p workdir --features http-client`: **156 passed**, doc tests passed.
Logs: `target/t743-combined-workdir.log`, `target/t743-combined-workdir-http.log`.
- Combined Web check: **0 errors, 0 warnings**; static production build passed
with the existing chunk-size advisory. Web tests **613 passed**, components
**318 passed**, Chromium Ticket/principal browser tests **7 passed**.
Logs: `target/t743-combined-{web-check,web-build,web-tests,components,browser}.log`.
The last change is Server execution placement/test code, not a UI change.
- All six server-api TypeScript generator `--check` commands passed: ticket,
repository OpenAPI, runtime, workdir, Worker launch and companion.
- `cargo fmt --all -- --check`, `git diff --check HEAD` and
`env -u LD_LIBRARY_PATH deno run --allow-read --allow-run=git tools/check-state-contract-names.ts`
passed; naming: **558 matching lines, 0 unapproved, 0 stale exceptions**.
Earlier T-743 public-surface, five-crate, Prompt/Tool, visual and real-Worker
fixture evidence is retained in `T-743-public-surface-validation.md`. Scripted
model/loopback browser fixtures do not attest real model judgment or all T-744
live paths. Target integration, Ticket/MR conclusion, live update and retained
Worker/Workdir cleanup were not performed. Fresh independent combined-source
review remains pending the upstream fixes, not silently waived.
@@ -26,7 +26,7 @@ Canonical `/workspaces/{workspace_id}/tickets/{ticket_id}` and `/api/w/{workspac
## Validation
Passed on the combined working snapshot:
Passed on the initial T-743 public-surface snapshot (before T-748 composition); later combined execution-boundary evidence is in `T-743-T748-composition.md`:
- Root `cargo check` (normal TUI/Server compile closure).
- `cargo test --no-fail-fast -p ticket -p worker-runtime -p worker -p yoi-workspace-server -p server-api`: all selected crate targets passed. Library counts: server-api 98; ticket 99; worker 994; worker-runtime 417; workspace-server 946 with one existing ignored test. Included integration/binary targets also passed. Log: `target/t743-changed-crates-final.log`.
@@ -51,7 +51,7 @@ List/Orchestrator final evidence: `t743-list-final-reviewed-light` and `t743-lis
## Explicit remaining work and limits
T-748's DB execution facade is **not** copied or integrated. Its latest notified candidate is local-only and its full Server suite has unresolved failures. When T-748 publishes a validated source, normal-merge it, place each new whole synchronous domain helper/authority query into the common bounded DB execution boundary, keep the answer transaction intact, and leave Runtime/network waiting off the DB worker. Recheck cancellation/overload and commit-before-publication at the combined boundary. The exact combined published source then needs independent trusted review.
The published T-748 source `66bb9d0b825d72e20923003b273b787007f3aed8` has now been normally merged into the same T-743 source and the added callers connected to its existing bounded owner. The answer transaction remains intact; Runtime live discovery and post-commit dispatch use the finite Host lane. Combined full Server/Workdir and ten new boundary regressions pass; see `T-743-T748-composition.md` for exact proof and distinctions. However T-748 MR seq3 has two unresolved major findings on other Subjektiv/network and cold protocol signing paths. A validated upstream fix source, normal composition/revalidation and fresh independent exact combined-source review remain necessary. This is not final approval.
Current evidence does not attest actual model judgment quality. Prompt/Tool fixtures are scripted inclusion/schema/transport tests; Chromium uses a loopback API fixture. The real-Worker Server fixture uses a scripted model with actual Controller/session/Runtime acceptance and canonical processing synchronization. None is a live migration/rollout, proof of actual processing for every production delivery, or completion of T-744's full real-path validation.