merge: compose T748 bounded execution with T743 public callers
This commit is contained in:
+145
-16
@@ -432,16 +432,23 @@ mod client {
|
||||
use super::*;
|
||||
use crate::{Workdir, WorkdirSession};
|
||||
|
||||
/// Provides a fresh bearer token for each Runtime request. Backend
|
||||
/// implementations can mint short-lived capability tokens without making a
|
||||
/// Worker-bound session expire with the token used to open it.
|
||||
/// An owned, Send authorization future that can offload synchronous authority
|
||||
/// work without borrowing request data or the authorization provider.
|
||||
pub type WorkdirHttpAuthorizationFuture = std::pin::Pin<
|
||||
Box<dyn std::future::Future<Output = Result<String, WorkdirError>> + Send + 'static>,
|
||||
>;
|
||||
|
||||
/// Provides a fresh bearer token asynchronously for each Runtime request.
|
||||
/// Backend implementations can mint short-lived capability tokens without
|
||||
/// making a Worker-bound session expire with the token used to open it.
|
||||
/// Blocking DB/signing work must be offloaded by the implementation.
|
||||
pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: &str,
|
||||
path_and_query: &str,
|
||||
body: &[u8],
|
||||
) -> Result<String, WorkdirError>;
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture;
|
||||
}
|
||||
|
||||
struct FixedBearerToken(Arc<str>);
|
||||
@@ -455,11 +462,12 @@ mod client {
|
||||
impl WorkdirHttpAuthorization for FixedBearerToken {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
_method: &str,
|
||||
_path_and_query: &str,
|
||||
_body: &[u8],
|
||||
) -> Result<String, WorkdirError> {
|
||||
Ok(self.0.to_string())
|
||||
_method: String,
|
||||
_path_and_query: String,
|
||||
_body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let token = self.0.clone();
|
||||
Box::pin(async move { Ok(token.to_string()) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,7 +517,9 @@ mod client {
|
||||
)?;
|
||||
let body = serde_json::to_vec(&request)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
|
||||
let token = authorization.bearer_token("POST", url.path(), &body)?;
|
||||
let token = authorization
|
||||
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
|
||||
.await?;
|
||||
let response = client
|
||||
.post(url)
|
||||
.bearer_auth(token)
|
||||
@@ -560,7 +570,10 @@ mod client {
|
||||
let operation = WorkdirSessionOperationRequest { operation };
|
||||
let body = serde_json::to_vec(&operation)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
|
||||
let token = self.authorization.bearer_token("POST", url.path(), &body)?;
|
||||
let token = self
|
||||
.authorization
|
||||
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
|
||||
.await?;
|
||||
let response = self
|
||||
.client
|
||||
.post(url)
|
||||
@@ -864,7 +877,10 @@ mod client {
|
||||
&self.base_url,
|
||||
&["v1", "workdir-sessions", self.session_id.as_str()],
|
||||
)?;
|
||||
let token = self.authorization.bearer_token("DELETE", url.path(), &[])?;
|
||||
let token = self
|
||||
.authorization
|
||||
.bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new())
|
||||
.await?;
|
||||
let response = self
|
||||
.client
|
||||
.delete(url)
|
||||
@@ -918,11 +934,124 @@ mod client {
|
||||
WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod authorization_tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct RejectAuthorization {
|
||||
requests: Arc<Mutex<Vec<(String, String, Vec<u8>)>>>,
|
||||
}
|
||||
|
||||
impl WorkdirHttpAuthorization for RejectAuthorization {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let requests = self.requests.clone();
|
||||
Box::pin(async move {
|
||||
tokio::task::yield_now().await;
|
||||
let mut requests = requests.lock().unwrap();
|
||||
requests.push((method, path_and_query, body));
|
||||
Err(WorkdirError::Unavailable(format!(
|
||||
"authorization rejected request {}",
|
||||
requests.len()
|
||||
)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let base_url: Url = format!("http://{}", listener.local_addr().unwrap())
|
||||
.parse()
|
||||
.unwrap();
|
||||
let client = Client::builder().no_proxy().build().unwrap();
|
||||
let authorization = Arc::new(RejectAuthorization::default());
|
||||
let workdir = Workdir::new("w");
|
||||
let open_request = OpenWorkdirSessionRequest {
|
||||
owner_worker_id: Some("worker".to_string()),
|
||||
};
|
||||
let error = RemoteWorkdirSession::open_with_authorization(
|
||||
client.clone(),
|
||||
base_url.clone(),
|
||||
authorization.clone(),
|
||||
workdir.id().clone(),
|
||||
open_request.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 1"
|
||||
));
|
||||
|
||||
// Seed an already-open session to exercise the public operation and
|
||||
// close boundaries independently of the rejected open request.
|
||||
let remote = RemoteWorkdirSession {
|
||||
client,
|
||||
base_url,
|
||||
authorization: authorization.clone(),
|
||||
workdir,
|
||||
session_id: WorkdirSessionId::new("s").unwrap(),
|
||||
capabilities: WorkdirSessionCapabilities::EMPTY,
|
||||
closed: AtomicBool::new(false),
|
||||
};
|
||||
let stat = StatRequest {
|
||||
path: crate::WorkdirPath::new("file").unwrap(),
|
||||
};
|
||||
let error = remote.stat(stat.clone()).await.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 2"
|
||||
));
|
||||
let error = remote.close().await.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 3"
|
||||
));
|
||||
assert_eq!(
|
||||
*authorization.requests.lock().unwrap(),
|
||||
vec![
|
||||
(
|
||||
"POST".to_string(),
|
||||
"/v1/working-directories/w/sessions".to_string(),
|
||||
serde_json::to_vec(&open_request).unwrap(),
|
||||
),
|
||||
(
|
||||
"POST".to_string(),
|
||||
"/v1/workdir-sessions/s/operations".to_string(),
|
||||
serde_json::to_vec(&WorkdirSessionOperationRequest {
|
||||
operation: WorkdirSessionOperation::Stat(stat),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
(
|
||||
"DELETE".to_string(),
|
||||
"/v1/workdir-sessions/s".to_string(),
|
||||
Vec::new(),
|
||||
),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
listener.accept().unwrap_err().kind(),
|
||||
std::io::ErrorKind::WouldBlock
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub use self::RemoteWorkdirSession as ClientSession;
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-client")]
|
||||
pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization};
|
||||
pub use client::{
|
||||
ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
@@ -147,7 +147,7 @@ async fn assignment_delivery_response_loss_returns_immutable_payload_after_progr
|
||||
.update_ticket_assignment_tracking("w", "tracking", &update, &source)
|
||||
.unwrap();
|
||||
drop(store);
|
||||
let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.list_pending_ticket_assignment_deliveries("w", 100)
|
||||
@@ -208,7 +208,7 @@ async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt()
|
||||
let path = directory.path().join("workspace.db");
|
||||
let input = pending.input_event_id.clone();
|
||||
std::thread::spawn(move || {
|
||||
let store = SqliteWorkspaceStore::open(path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
|
||||
barrier.wait();
|
||||
store
|
||||
.reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source())
|
||||
@@ -568,7 +568,7 @@ async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_co
|
||||
"original rendered prompt"
|
||||
);
|
||||
drop(store);
|
||||
let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.prepare_ticket_assignment_delivery_content(
|
||||
@@ -634,7 +634,7 @@ fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_l
|
||||
let operations = stored_fixture_rows(&conn, "ticket_assignment_operations");
|
||||
let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history");
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let pending = store
|
||||
.list_pending_ticket_assignment_deliveries("w", 100)
|
||||
.unwrap();
|
||||
@@ -684,7 +684,7 @@ async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once
|
||||
let barrier = barrier.clone();
|
||||
let path = directory.path().join("workspace.db");
|
||||
std::thread::spawn(move || {
|
||||
let store = SqliteWorkspaceStore::open(path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
|
||||
barrier.wait();
|
||||
delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source())
|
||||
.unwrap()
|
||||
@@ -716,8 +716,8 @@ fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() {
|
||||
let conn = tracking_schema91_fixture(&path);
|
||||
migrate_ticket_tracking_v91_to_v92(&conn).unwrap();
|
||||
drop(conn);
|
||||
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
let objects = |store: &SqliteWorkspaceStore| {
|
||||
store.with_conn(|conn| {
|
||||
let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?;
|
||||
@@ -741,7 +741,7 @@ async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() {
|
||||
.unwrap();
|
||||
drop(store);
|
||||
assert!(
|
||||
matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update"))
|
||||
matches!(SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update"))
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ async fn tracking_store_every_replay_fences_current_singleton_and_authenticated_
|
||||
);
|
||||
// Model the authority change after authenticated source capture, before
|
||||
// BEGIN IMMEDIATE on a second connection. Receipt presence cannot bypass it.
|
||||
let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
second.with_conn_mut(|conn| {
|
||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
match change {
|
||||
@@ -233,7 +233,7 @@ async fn tracking_store_same_principal_reissue_and_exact_retries_keep_original_r
|
||||
.is_err()
|
||||
);
|
||||
drop(store);
|
||||
let reopened = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.get_ticket_assignment_tracking("w", "tracking", "a")
|
||||
@@ -441,7 +441,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
.collect::<Vec<_>>(),
|
||||
vec![92, 93]
|
||||
);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, 93);
|
||||
@@ -525,7 +525,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
.list_ticket_assignment_tracking_history("w", "pending", "a-pending")
|
||||
.unwrap();
|
||||
drop(store);
|
||||
let reopened = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.list_ticket_assignment_tracking_history("w", "pending", "a-pending")
|
||||
@@ -550,7 +550,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
|
||||
let releases_before = stored_fixture_rows(&conn, "ticket_assignment_work_releases");
|
||||
drop(conn);
|
||||
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let backend = ticket::SqliteTicketBackend::open_verified(&path, "w").unwrap();
|
||||
let reopen = || {
|
||||
let observed = backend.show("historical-done".into()).unwrap();
|
||||
@@ -645,7 +645,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
|
||||
);
|
||||
// A second observer holds the same Assign baseline. Only the first
|
||||
// explicit writer may consume it; BEGIN IMMEDIATE serializes the CAS.
|
||||
let second = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let second = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
second
|
||||
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
|
||||
@@ -740,7 +740,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
|
||||
}).unwrap();
|
||||
drop(second);
|
||||
drop(store);
|
||||
let reopened_store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let reopened_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
reopened_store
|
||||
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
|
||||
@@ -796,7 +796,7 @@ fn tracking_store_schema91_missing_assign_audit_remains_unknown_without_cas_base
|
||||
)
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let unknown = store
|
||||
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
|
||||
.unwrap()
|
||||
@@ -1068,7 +1068,7 @@ async fn orchestrator_assignment_all_runtime_retries_fence_singleton_and_binding
|
||||
&source,
|
||||
)
|
||||
.unwrap();
|
||||
let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
second.with_conn_mut(|conn| {let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;match change {
|
||||
"replace" => {tx.execute("UPDATE worker_singleton_owners SET worker_id='replacement',generation=2 WHERE workspace_id='w'",[])?;},
|
||||
"revoke" => {tx.execute("UPDATE workspace_runtime_bindings SET revoked_at='revoked' WHERE workspace_id='w'",[])?;},
|
||||
|
||||
@@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option<String> {
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
|
||||
store: &S,
|
||||
pub async fn resolve_request_actor(
|
||||
store: std::sync::Arc<dyn ControlPlaneStore>,
|
||||
headers: &HeaderMap,
|
||||
cookie_name: &str,
|
||||
) -> Result<Option<RequestActor>> {
|
||||
if let Some(token) = parse_bearer(headers) {
|
||||
let bearer = parse_bearer(headers);
|
||||
let session = parse_cookie(headers, cookie_name);
|
||||
if bearer.is_none() && session.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let executor = store.db_executor();
|
||||
executor
|
||||
.run(move || resolve_request_actor_sync(store.as_ref(), bearer, session))
|
||||
.await
|
||||
}
|
||||
|
||||
// One owned synchronous stage includes token usage marking and the user read.
|
||||
// Do not call async store wrappers here: they use the same executor themselves.
|
||||
fn resolve_request_actor_sync(
|
||||
store: &dyn ControlPlaneStore,
|
||||
bearer: Option<String>,
|
||||
session: Option<String>,
|
||||
) -> Result<Option<RequestActor>> {
|
||||
if let Some(token) = bearer {
|
||||
let token_hash = token_hash(&token);
|
||||
if let Some(api_token) = store.resolve_api_token(&token_hash)? {
|
||||
if api_token
|
||||
@@ -109,7 +127,7 @@ pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(session_token) = parse_cookie(headers, cookie_name) {
|
||||
if let Some(session_token) = session {
|
||||
if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? {
|
||||
if is_expired(&session.expires_at) {
|
||||
return Ok(None);
|
||||
@@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::{
|
||||
AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn seeded_store() -> Arc<SqliteWorkspaceStore> {
|
||||
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::in_memory_for_test()
|
||||
.unwrap()
|
||||
.with_db_executor(executor),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-auth".into(),
|
||||
kind: "user".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_user(&UserRecord {
|
||||
user_id: "user-auth".into(),
|
||||
account_id: "account-auth".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.create_browser_session(&BrowserSessionRecord {
|
||||
session_id: "session-auth".into(),
|
||||
token_hash: token_hash("session-secret"),
|
||||
user_id: "user-auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: "9999-01-01T00:00:00Z".into(),
|
||||
revoked_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() {
|
||||
let store = seeded_store();
|
||||
let hash = token_hash("api-secret");
|
||||
store
|
||||
.create_api_token(&ApiTokenRecord {
|
||||
token_id: "token-auth".into(),
|
||||
token_hash: hash.clone(),
|
||||
user_id: "user-auth".into(),
|
||||
label: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: None,
|
||||
revoked_at: None,
|
||||
last_used_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap());
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let actor = resolve_request_actor(store.clone(), &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(actor.user_id, "user-auth");
|
||||
assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken);
|
||||
assert!(
|
||||
store
|
||||
.resolve_api_token(&hash)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.last_used_at
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_api_token_does_not_fall_back_to_valid_browser_session() {
|
||||
let store = seeded_store();
|
||||
let hash = token_hash("expired-secret");
|
||||
store
|
||||
.create_api_token(&ApiTokenRecord {
|
||||
token_id: "token-auth".into(),
|
||||
token_hash: hash.clone(),
|
||||
user_id: "user-auth".into(),
|
||||
label: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: Some("2000-01-01T00:00:00Z".into()),
|
||||
revoked_at: None,
|
||||
last_used_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::AUTHORIZATION,
|
||||
"Bearer expired-secret".parse().unwrap(),
|
||||
);
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
assert!(
|
||||
resolve_request_actor(store.clone(), &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.resolve_api_token(&hash)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.last_used_at
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_bearer_can_fall_back_to_valid_browser_session() {
|
||||
let store = seeded_store();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::AUTHORIZATION,
|
||||
"Bearer unknown-secret".parse().unwrap(),
|
||||
);
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let actor = resolve_request_actor(store, &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(actor.user_id, "user-auth");
|
||||
assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() {
|
||||
let store = seeded_store();
|
||||
let executor = store.db_executor();
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker_executor = executor.clone();
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_executor
|
||||
.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
release_rx.recv().unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) }));
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(1),
|
||||
resolve_request_actor(store, &headers, "session"),
|
||||
)
|
||||
.await;
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap().unwrap();
|
||||
queued.await.unwrap();
|
||||
assert!(matches!(
|
||||
result.expect("capacity rejection must not wait for queued SQL"),
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_code_uses_the_uuid_random_tail() {
|
||||
|
||||
@@ -238,6 +238,13 @@ impl SqliteWorkspaceAuthority {
|
||||
})
|
||||
}
|
||||
|
||||
// Contention fixtures must hold the candidate query's actual connection,
|
||||
// not the API Store used only by some enrichment paths.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn candidate_query_store_for_test(&self) -> SqliteWorkspaceStore {
|
||||
self.store.clone()
|
||||
}
|
||||
|
||||
/// Assignment self-report enrichment is a separate projection so retained
|
||||
/// responsibility and existing Ticket constructors stay distinct from status.
|
||||
pub fn ticket_assignment_tracking(
|
||||
@@ -3349,7 +3356,7 @@ mod tests {
|
||||
async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "w".into(),
|
||||
@@ -3424,7 +3431,7 @@ mod tests {
|
||||
async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "w".into(),
|
||||
@@ -3557,7 +3564,7 @@ mod tests {
|
||||
write_ticket(dir.path(), "00000000001J5", "Second ticket", "queued");
|
||||
write_ticket(dir.path(), "00000000001J6", "Third ticket", "planning");
|
||||
let db_path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-test".to_string(),
|
||||
@@ -3989,7 +3996,7 @@ VALUES ('workspace-test', 'ticket', 4);
|
||||
async fn objective_mutations_write_sqlite_records_and_audit_events() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db_path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-test".to_string(),
|
||||
|
||||
@@ -795,7 +795,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn open_store() -> SqliteWorkspaceStore {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
store
|
||||
}
|
||||
@@ -1454,7 +1454,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_candidate_is_never_persisted() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let main = current.snapshot.get(&path(MAIN_CONFIG_ENTRYPOINT)).unwrap();
|
||||
@@ -1478,7 +1478,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn valid_candidate_commits_snapshot_content_and_provenance_atomically() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let committed = store
|
||||
@@ -1496,7 +1496,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_cas_cannot_overwrite_newer_tree() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let request = commit_request(¤t, vec![update_main(¤t, "{ answer = 42; }")]);
|
||||
@@ -1514,7 +1514,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_content_remains_retrievable_after_later_commit() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let first = store
|
||||
@@ -1772,7 +1772,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn migration_creates_config_authority_tables() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
for table in [
|
||||
|
||||
@@ -0,0 +1,957 @@
|
||||
//! Bounded execution of whole synchronous DB operations, independent of Tokio.
|
||||
//!
|
||||
//! Accepted jobs belong to the workers, not their async waiters. Cancelling a
|
||||
//! waiter does not cancel or retry SQL, release capacity, or release its lease.
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
future::Future,
|
||||
num::NonZeroUsize,
|
||||
panic::{AssertUnwindSafe, catch_unwind},
|
||||
sync::{Arc, Mutex, OnceLock, mpsc},
|
||||
};
|
||||
use tokio::sync::{Notify, oneshot};
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// The default is one FIFO worker and at most 32 waiting operations.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct DbExecutorLimits {
|
||||
pub workers: NonZeroUsize,
|
||||
pub queued: NonZeroUsize,
|
||||
}
|
||||
|
||||
impl Default for DbExecutorLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(32).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A request can retain its lifecycle authority until actual SQL finishes.
|
||||
/// Retention happens at submission, while the caller's task-local scope exists.
|
||||
pub(crate) trait DbOperationLifetime: Send + Sync {
|
||||
fn retain(&self) -> Box<dyn Send>;
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static OPERATION_LIFETIME: Arc<dyn DbOperationLifetime>;
|
||||
}
|
||||
|
||||
pub(crate) async fn with_operation_lifetime<F: Future>(
|
||||
lifetime: Arc<dyn DbOperationLifetime>,
|
||||
future: F,
|
||||
) -> F::Output {
|
||||
OPERATION_LIFETIME.scope(lifetime, future).await
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Accepted {
|
||||
last_job_id: u64,
|
||||
unfinished: BTreeSet<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Tracking {
|
||||
accepted: Mutex<Accepted>,
|
||||
finished: Notify,
|
||||
}
|
||||
|
||||
struct Completion {
|
||||
tracking: Arc<Tracking>,
|
||||
id: u64,
|
||||
}
|
||||
|
||||
impl Drop for Completion {
|
||||
fn drop(&mut self) {
|
||||
self.tracking
|
||||
.accepted
|
||||
.lock()
|
||||
.unwrap()
|
||||
.unfinished
|
||||
.remove(&self.id);
|
||||
self.tracking.finished.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
struct Job {
|
||||
operation: Box<dyn FnOnce() + Send>,
|
||||
// Field drop order releases authority before completion becomes observable.
|
||||
_lifetime: Option<Box<dyn Send>>,
|
||||
lifetime_source: Option<Arc<dyn DbOperationLifetime>>,
|
||||
_completion: Completion,
|
||||
runtime: Option<tokio::runtime::Handle>,
|
||||
diagnostics: tracing::Dispatch,
|
||||
span: tracing::Span,
|
||||
}
|
||||
|
||||
fn report_unknown(accepted_job_id: u64, phase: &'static str) {
|
||||
tracing::error!(
|
||||
code = "db_operation_outcome_unknown",
|
||||
accepted_job_id,
|
||||
phase,
|
||||
error = %Error::DbOperationOutcomeUnknown,
|
||||
"DB job panicked; outcome unknown, operation will not be retried"
|
||||
);
|
||||
}
|
||||
|
||||
/// Clones share the queue and actual-job budget. Workers retain only the
|
||||
/// receiver, so dropping the last sender closes the queue; accepted jobs finish
|
||||
/// before the dedicated threads exit. No thread is created per store open.
|
||||
#[derive(Clone)]
|
||||
pub struct DbExecutor {
|
||||
sender: Option<mpsc::SyncSender<Job>>,
|
||||
tracking: Arc<Tracking>,
|
||||
max_jobs: usize,
|
||||
}
|
||||
|
||||
impl DbExecutor {
|
||||
pub fn new(limits: DbExecutorLimits) -> Result<Self> {
|
||||
let max_jobs = limits
|
||||
.workers
|
||||
.get()
|
||||
.checked_add(limits.queued.get())
|
||||
.ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?;
|
||||
let (sender, receiver) = mpsc::sync_channel::<Job>(limits.queued.get());
|
||||
let receiver = Arc::new(Mutex::new(receiver));
|
||||
for index in 0..limits.workers.get() {
|
||||
let receiver = receiver.clone();
|
||||
std::thread::Builder::new()
|
||||
.name(format!("workspace-db-{index}"))
|
||||
.spawn(move || {
|
||||
loop {
|
||||
let next = receiver.lock().unwrap().recv();
|
||||
let Ok(job) = next else { break };
|
||||
let Job {
|
||||
operation,
|
||||
_lifetime: lifetime,
|
||||
lifetime_source,
|
||||
_completion: completion,
|
||||
runtime,
|
||||
diagnostics,
|
||||
span,
|
||||
} = job;
|
||||
// Keep diagnostics with their submitting subscriber as
|
||||
// well as the runtime; no process-global override.
|
||||
let _diagnostics = tracing::dispatcher::set_default(&diagnostics);
|
||||
let _span = span.enter();
|
||||
// Preserve the submitting runtime for synchronous Host
|
||||
// post-commit hooks, including reply/lease destructors.
|
||||
// enter() supplies context only: never drive a future
|
||||
// or await/nest an executor submission in a transaction.
|
||||
let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter);
|
||||
// A cancelled waiter's discarded result may itself have
|
||||
// a destructor. Keep both the lease and actual-job slot
|
||||
// until that cleanup finishes, even if it panics.
|
||||
let execute = || {
|
||||
if catch_unwind(AssertUnwindSafe(operation)).is_err() {
|
||||
report_unknown(completion.id, "reply_cleanup");
|
||||
}
|
||||
if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() {
|
||||
report_unknown(completion.id, "lifetime_cleanup");
|
||||
}
|
||||
};
|
||||
// Synchronous post-commit continuations can admit another
|
||||
// bounded job. Transfer the original lifecycle source as
|
||||
// well, so its authority survives the parent job/waiter.
|
||||
// This does not inject context into detached Tokio tasks.
|
||||
if let Some(source) = lifetime_source {
|
||||
OPERATION_LIFETIME.sync_scope(source, execute);
|
||||
} else {
|
||||
execute();
|
||||
}
|
||||
drop(completion);
|
||||
}
|
||||
})
|
||||
.map_err(|_| Error::DbExecutorUnavailable)?;
|
||||
}
|
||||
Ok(Self {
|
||||
sender: Some(sender),
|
||||
tracking: Arc::default(),
|
||||
max_jobs,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn shared() -> Self {
|
||||
static SHARED: OnceLock<DbExecutor> = OnceLock::new();
|
||||
SHARED
|
||||
.get_or_init(|| {
|
||||
Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self {
|
||||
sender: None,
|
||||
tracking: Arc::default(),
|
||||
max_jobs: 0,
|
||||
})
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Submit without waiting for queue capacity. The closure must contain the
|
||||
/// entire synchronous operation/transaction, with no nested async runtime.
|
||||
/// The submitting Tokio handle, when present, is entered for synchronous
|
||||
/// post-commit hooks and worker cleanup; it is never used to block_on SQL.
|
||||
pub async fn run<T, E>(
|
||||
&self,
|
||||
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
|
||||
) -> std::result::Result<T, E>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: From<Error> + Send + 'static,
|
||||
{
|
||||
self.submit(operation)?
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into()))
|
||||
}
|
||||
|
||||
/// Admit a fire-and-forget operation immediately, without spawning a Tokio
|
||||
/// task or waiting for capacity. Err means the operation was not accepted;
|
||||
/// Ok means the worker owns it through execution and cleanup, even after
|
||||
/// the submitting task is cancelled. A panic emits a structured unknown-
|
||||
/// outcome diagnostic, never an automatic retry. drain() tracks these jobs.
|
||||
pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> {
|
||||
self.submit(move || {
|
||||
operation();
|
||||
Ok::<_, Error>(())
|
||||
})
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
/// Shared synchronous admission: neither dropped receivers nor cancelled
|
||||
/// callers remove accepted work from the queue or the actual-job budget.
|
||||
fn submit<T, E>(
|
||||
&self,
|
||||
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
|
||||
) -> std::result::Result<oneshot::Receiver<std::result::Result<T, E>>, E>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: From<Error> + Send + 'static,
|
||||
{
|
||||
let sender = self
|
||||
.sender
|
||||
.as_ref()
|
||||
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
|
||||
let runtime = tokio::runtime::Handle::try_current().ok();
|
||||
let diagnostics = tracing::dispatcher::get_default(Clone::clone);
|
||||
let span = tracing::Span::current();
|
||||
let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok();
|
||||
let lifetime = lifetime_source.as_ref().map(|source| source.retain());
|
||||
let (reply, result) = oneshot::channel();
|
||||
// Serialize admission and acceptance snapshots. Worker completion cannot
|
||||
// race acceptance, and cancelled waiters never alter this actual-job cap.
|
||||
{
|
||||
let mut accepted = self.tracking.accepted.lock().unwrap();
|
||||
if accepted.unfinished.len() >= self.max_jobs {
|
||||
return Err(Error::DbCapacityExceeded.into());
|
||||
}
|
||||
let id = accepted
|
||||
.last_job_id
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
|
||||
// Never reuse the identity of a rejected job: its destructor may run
|
||||
// after another submission has acquired the admission lock.
|
||||
accepted.last_job_id = id;
|
||||
let job = Job {
|
||||
operation: Box::new(move || {
|
||||
let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| {
|
||||
report_unknown(id, "operation");
|
||||
Err(Error::DbOperationOutcomeUnknown.into())
|
||||
});
|
||||
let _ = reply.send(outcome);
|
||||
}),
|
||||
_lifetime: lifetime,
|
||||
lifetime_source,
|
||||
runtime,
|
||||
diagnostics,
|
||||
span,
|
||||
_completion: Completion {
|
||||
tracking: self.tracking.clone(),
|
||||
id,
|
||||
},
|
||||
};
|
||||
match sender.try_send(job) {
|
||||
Ok(()) => {
|
||||
accepted.unfinished.insert(id);
|
||||
}
|
||||
Err(error) => {
|
||||
// The rejected Job's completion also locks tracking; drop it only
|
||||
// after releasing the admission lock.
|
||||
drop(accepted);
|
||||
return Err(match error {
|
||||
mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded,
|
||||
mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Wait for every job accepted before this barrier is first polled (not
|
||||
/// later submissions). This bypasses queue capacity and survives cancelled
|
||||
/// job waiters. It does not close the queue or stop new submissions.
|
||||
pub async fn drain(&self) -> Result<()> {
|
||||
if self.sender.is_none() {
|
||||
return Err(Error::DbExecutorUnavailable);
|
||||
}
|
||||
let through = self.tracking.accepted.lock().unwrap().last_job_id;
|
||||
loop {
|
||||
let notified = self.tracking.finished.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self
|
||||
.tracking
|
||||
.accepted
|
||||
.lock()
|
||||
.unwrap()
|
||||
.unfinished
|
||||
.first()
|
||||
.is_none_or(|id| *id > through)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rusqlite::{Connection, TransactionBehavior, ffi};
|
||||
use std::{ffi::c_void, time::Duration};
|
||||
|
||||
fn executor(queued: usize) -> DbExecutor {
|
||||
DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(queued).unwrap(),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn database() -> (tempfile::TempDir, Connection, Connection) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("jobs.sqlite");
|
||||
let owner = Connection::open(&path).unwrap();
|
||||
owner
|
||||
.execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)")
|
||||
.unwrap();
|
||||
let worker = Connection::open(&path).unwrap();
|
||||
(dir, owner, worker)
|
||||
}
|
||||
|
||||
struct BusyGate {
|
||||
waiting: Option<oneshot::Sender<()>>,
|
||||
release: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
// A fixture-owned callback proves SQLite actually reached its lock wait.
|
||||
// Unlike busy_handler's fn pointer API, the SQLite API carries a per-call
|
||||
// context, so tests need neither process-global state nor guessed sleeps.
|
||||
struct BusyRegistration {
|
||||
db: *mut ffi::sqlite3,
|
||||
_gate: Box<BusyGate>,
|
||||
}
|
||||
|
||||
impl BusyRegistration {
|
||||
fn install(conn: &Connection, gate: BusyGate) -> Self {
|
||||
unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 {
|
||||
// The Box remains registered only while this connection and
|
||||
// BusyRegistration are alive, on this same dedicated worker.
|
||||
let gate = unsafe { &mut *context.cast::<BusyGate>() };
|
||||
if let Some(waiting) = gate.waiting.take() {
|
||||
let _ = waiting.send(());
|
||||
if gate.release.recv().is_ok() {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
let mut gate = Box::new(gate);
|
||||
// SAFETY: conn owns the handle; registration is dropped before conn.
|
||||
let db = unsafe { conn.handle() };
|
||||
let result = unsafe {
|
||||
ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast())
|
||||
};
|
||||
assert_eq!(result, ffi::SQLITE_OK);
|
||||
Self { db, _gate: gate }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BusyRegistration {
|
||||
fn drop(&mut self) {
|
||||
// SAFETY: unregister before releasing the callback's boxed context.
|
||||
unsafe {
|
||||
ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn observed<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::time::timeout(Duration::from_secs(5), future)
|
||||
.await
|
||||
.expect("DB job did not reach the expected observable state")
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() {
|
||||
let (_dir, mut owner, mut worker) = database();
|
||||
let lock = owner
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let executor = executor(1);
|
||||
let (waiting, blocked) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut write = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _busy = BusyRegistration::install(
|
||||
&worker,
|
||||
BusyGate {
|
||||
waiting: Some(waiting),
|
||||
release: permit,
|
||||
},
|
||||
);
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
observed(blocked).await.unwrap();
|
||||
let control = tokio::spawn(async { 42 });
|
||||
assert_eq!(observed(control).await.unwrap(), 42);
|
||||
// The timer itself is the contract under test, not synchronization.
|
||||
observed(tokio::time::sleep(Duration::from_millis(1))).await;
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
lock.rollback().unwrap();
|
||||
release.send(()).unwrap();
|
||||
observed(write).await.unwrap();
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
struct Lease(Arc<std::sync::atomic::AtomicUsize>);
|
||||
struct Retained(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl DbOperationLifetime for Lease {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Box::new(Retained(self.0.clone()))
|
||||
}
|
||||
}
|
||||
impl Drop for Retained {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() {
|
||||
let (_dir, mut owner, mut worker) = database();
|
||||
let lock = owner
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let executor = executor(1);
|
||||
let (waiting, blocked) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut running = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _busy = BusyRegistration::install(
|
||||
&worker,
|
||||
BusyGate {
|
||||
waiting: Some(waiting),
|
||||
release: permit,
|
||||
},
|
||||
);
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut running).is_pending());
|
||||
observed(blocked).await.unwrap();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap();
|
||||
let mut queued = Box::pin(with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<()> {
|
||||
let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
// FIFO is observable in durable SQL, not inferred from waiter order.
|
||||
let first: i64 =
|
||||
tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| {
|
||||
row.get(0)
|
||||
})?;
|
||||
assert_eq!(first, 1);
|
||||
tx.execute("INSERT INTO writes VALUES (2)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
assert!(matches!(
|
||||
executor.run(|| Ok::<_, Error>(())).await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
let mut drained = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
lock.rollback().unwrap();
|
||||
release.send(()).unwrap();
|
||||
observed(running).await.unwrap();
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() {
|
||||
let (dir, owner, mut parent_conn) = database();
|
||||
let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap();
|
||||
let parent = executor(1);
|
||||
let child = executor(1);
|
||||
let child_owner = child.clone();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let source: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let (entered, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let (child_entered, child_running) = oneshot::channel();
|
||||
let (finish_child, child_permit) = mpsc::channel();
|
||||
let mut operation = Box::pin(with_operation_lifetime(
|
||||
source,
|
||||
parent.run(move || -> Result<()> {
|
||||
let _ = entered.send(());
|
||||
let _ = permit.recv();
|
||||
let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
child_owner.dispatch(move || {
|
||||
let tx = child_conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let committed: i64 = tx
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(committed, 1, "child must see its parent's committed SQL");
|
||||
let _ = child_entered.send(());
|
||||
let _ = child_permit.recv();
|
||||
tx.execute("INSERT INTO writes VALUES (2)", []).unwrap();
|
||||
tx.commit().unwrap();
|
||||
})?;
|
||||
Ok(())
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(operation);
|
||||
release.send(()).unwrap();
|
||||
observed(child_running).await.unwrap();
|
||||
observed(parent.drain()).await.unwrap();
|
||||
assert_eq!(
|
||||
count.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"accepted continuation lost authority when its parent finished"
|
||||
);
|
||||
let mut drained = Box::pin(child.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
finish_child.send(()).unwrap();
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let error = executor
|
||||
.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.execute("INSERT INTO writes VALUES (NULL)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::Sqlite(_)));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let error = executor
|
||||
.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
panic!("lost outcome after commit");
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let error = with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
panic!("lost outcome before commit");
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
struct DiscardedReplyHook {
|
||||
expected_runtime: tokio::runtime::Id,
|
||||
lease_count: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Option<oneshot::Sender<(bool, usize)>>,
|
||||
}
|
||||
|
||||
impl Drop for DiscardedReplyHook {
|
||||
fn drop(&mut self) {
|
||||
let runtime =
|
||||
tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context");
|
||||
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
|
||||
let same_runtime = runtime.id() == self.expected_runtime;
|
||||
let dropped = self.dropped.take().unwrap();
|
||||
tokio::spawn(async move {
|
||||
let _ = dropped.send((same_runtime, retained));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop()
|
||||
{
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut blocker).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
let expected_runtime = tokio::runtime::Handle::current().id();
|
||||
let (hook, published) = oneshot::channel();
|
||||
let (finish, finishing) = mpsc::channel();
|
||||
let (dropped, reply_dropped) = oneshot::channel();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let operation_count = count.clone();
|
||||
let mut queued = Box::pin(with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<_> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
// Model synchronous Host hooks: publish only after commit, checking
|
||||
// try_current and spawning without driving any future on this worker.
|
||||
let runtime = tokio::runtime::Handle::try_current()
|
||||
.expect("post-commit hook lost runtime context");
|
||||
assert_eq!(runtime.id(), expected_runtime);
|
||||
tokio::spawn(async move {
|
||||
let _ = hook.send(());
|
||||
});
|
||||
let _ = finishing.recv();
|
||||
Ok(DiscardedReplyHook {
|
||||
expected_runtime,
|
||||
lease_count: operation_count,
|
||||
dropped: Some(dropped),
|
||||
})
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
release.send(()).unwrap();
|
||||
observed(blocker).await.unwrap();
|
||||
observed(published).await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
finish.send(()).unwrap();
|
||||
assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1));
|
||||
observed(drain).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() {
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(2).unwrap(),
|
||||
queued: NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let mut releases = Vec::new();
|
||||
// Observe each worker executing before submitting the next operation,
|
||||
// so queue occupancy and overload are deterministic, not timing guesses.
|
||||
for _ in 0..2 {
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut operation = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(operation);
|
||||
releases.push(release);
|
||||
}
|
||||
let (finished, mut queued_finished) = oneshot::channel();
|
||||
let mut queued = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = finished.send(());
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
for _ in 0..8 {
|
||||
let rejected_runs = rejected_runs.clone();
|
||||
assert!(matches!(
|
||||
executor
|
||||
.run(move || {
|
||||
rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok::<_, Error>(())
|
||||
})
|
||||
.await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
}
|
||||
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert!(matches!(
|
||||
queued_finished.try_recv(),
|
||||
Err(oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
for release in releases {
|
||||
release.send(()).unwrap();
|
||||
}
|
||||
observed(drain).await.unwrap();
|
||||
observed(queued_finished).await.unwrap();
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct UnknownDiagnostic {
|
||||
code: Option<String>,
|
||||
phase: Option<String>,
|
||||
accepted_job_id: Option<u64>,
|
||||
}
|
||||
|
||||
impl tracing::field::Visit for UnknownDiagnostic {
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
match field.name() {
|
||||
"code" => self.code = Some(value.to_owned()),
|
||||
"phase" => self.phase = Some(value.to_owned()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
|
||||
if field.name() == "accepted_job_id" {
|
||||
self.accepted_job_id = Some(value);
|
||||
}
|
||||
}
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct DiagnosticCapture(Arc<Mutex<Vec<UnknownDiagnostic>>>);
|
||||
|
||||
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for DiagnosticCapture {
|
||||
fn on_event(
|
||||
&self,
|
||||
event: &tracing::Event<'_>,
|
||||
_: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let mut diagnostic = UnknownDiagnostic::default();
|
||||
event.record(&mut diagnostic);
|
||||
self.0.lock().unwrap().push(diagnostic);
|
||||
}
|
||||
}
|
||||
|
||||
struct DispatchCleanup {
|
||||
expected_runtime: tokio::runtime::Id,
|
||||
lease_count: Arc<std::sync::atomic::AtomicUsize>,
|
||||
started: Option<oneshot::Sender<(bool, usize)>>,
|
||||
release: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
impl Drop for DispatchCleanup {
|
||||
fn drop(&mut self) {
|
||||
// No panicking assertions during unwind: report observations to the
|
||||
// test, and gate cleanup so drain/lifetime behavior is observable.
|
||||
let same_runtime = tokio::runtime::Handle::try_current()
|
||||
.is_ok_and(|runtime| runtime.id() == self.expected_runtime);
|
||||
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
|
||||
if let Some(started) = self.started.take() {
|
||||
let _ = started.send((same_runtime, retained));
|
||||
}
|
||||
let _ = self.release.recv();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry()
|
||||
{
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut blocker).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(blocker); // Cancellation does not free its running slot.
|
||||
let expected_runtime = tokio::runtime::Handle::current().id();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let operation_count = count.clone();
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let operation_attempts = attempts.clone();
|
||||
let (cleanup_started, cleaning_up) = oneshot::channel();
|
||||
let (finish, finishing) = mpsc::channel();
|
||||
// Model multiple tracing contexts without setting a global subscriber.
|
||||
// This also prevents tracing's single-subscriber callsite optimization
|
||||
// from caching an unsubscribed parallel test's Interest::never here.
|
||||
let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry());
|
||||
let captured = DiagnosticCapture::default();
|
||||
let diagnostics =
|
||||
tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone()));
|
||||
with_operation_lifetime(lease, async {
|
||||
tracing::dispatcher::with_default(&diagnostics, || {
|
||||
executor.dispatch(move || {
|
||||
let _cleanup = DispatchCleanup {
|
||||
expected_runtime,
|
||||
lease_count: operation_count,
|
||||
started: Some(cleanup_started),
|
||||
release: finishing,
|
||||
};
|
||||
operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let tx = worker
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
tx.execute("INSERT INTO writes VALUES (1)", []).unwrap();
|
||||
tx.commit().unwrap();
|
||||
panic!("dispatched post-commit outcome lost");
|
||||
})
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(); // Dispatch is admitted synchronously; no waiter survives.
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let rejected_operation = rejected_runs.clone();
|
||||
assert!(matches!(
|
||||
executor.dispatch(move || {
|
||||
rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}),
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
release.send(()).unwrap();
|
||||
assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1));
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
finish.send(()).unwrap();
|
||||
observed(drain).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
{
|
||||
let events = captured.0.lock().unwrap();
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(
|
||||
events[0].code.as_deref(),
|
||||
Some("db_operation_outcome_unknown")
|
||||
);
|
||||
assert_eq!(events[0].phase.as_deref(), Some("operation"));
|
||||
assert!(events[0].accepted_job_id.is_some());
|
||||
}
|
||||
let (completed, completion) = oneshot::channel();
|
||||
executor
|
||||
.dispatch(move || {
|
||||
let _ = completed.send(());
|
||||
})
|
||||
.unwrap();
|
||||
observed(executor.drain()).await.unwrap();
|
||||
observed(completion).await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let job = store
|
||||
@@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let job = store
|
||||
|
||||
@@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
};
|
||||
let migrated = store
|
||||
@@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let archived = store
|
||||
@@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let service =
|
||||
@@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() {
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let service =
|
||||
|
||||
@@ -533,6 +533,7 @@ mod existing_workspace_provisioning {
|
||||
// This fixture contains only the schema-87 cutover authorities, not the
|
||||
// unrelated full Server schema. Exercise the real service/store methods.
|
||||
Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
})
|
||||
}
|
||||
@@ -644,6 +645,7 @@ mod existing_workspace_provisioning {
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())),
|
||||
});
|
||||
let service = WorkspaceSigningIdentityService::new(
|
||||
@@ -816,7 +818,7 @@ mod current_authority {
|
||||
use super::*;
|
||||
|
||||
fn binding_store() -> (SqliteWorkspaceStore, WorkspaceRuntimeBinding) {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
let runtime_key =
|
||||
worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-key").unwrap();
|
||||
store.with_conn(|conn| {
|
||||
|
||||
@@ -23,7 +23,10 @@ use std::{
|
||||
};
|
||||
use workdir::{
|
||||
Workdir, WorkdirError, WorkdirSessionHandle,
|
||||
http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization},
|
||||
http::{
|
||||
OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization,
|
||||
WorkdirHttpAuthorizationFuture,
|
||||
},
|
||||
};
|
||||
use worker_runtime::RuntimeWorkspaceScope;
|
||||
use worker_runtime::catalog::{
|
||||
@@ -3477,6 +3480,38 @@ impl WorkspaceRuntimeAuthorization {
|
||||
}
|
||||
}
|
||||
|
||||
/// Offload the complete live-binding check and request signing operation.
|
||||
/// Claims are minted inside the bounded job, not cached before admission.
|
||||
pub(crate) async fn issue_async(
|
||||
&self,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
operation: String,
|
||||
worker_id: Option<String>,
|
||||
body: Vec<u8>,
|
||||
) -> Result<String, RuntimeDiagnostic> {
|
||||
let executor = self.store.db_executor();
|
||||
let authorization = self.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
Ok::<_, Error>(authorization.issue(
|
||||
&method,
|
||||
&path_and_query,
|
||||
&operation,
|
||||
worker_id.as_deref(),
|
||||
&body,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
diagnostic(
|
||||
"workspace_runtime_authorization_unavailable",
|
||||
HostDiagnosticSeverity::Error,
|
||||
error.to_string(),
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
pub(crate) fn issue(
|
||||
&self,
|
||||
method: &str,
|
||||
@@ -3649,25 +3684,24 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization {
|
||||
impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: &str,
|
||||
path_and_query: &str,
|
||||
body: &[u8],
|
||||
) -> Result<String, WorkdirError> {
|
||||
if let Some(authorization) = &self.workspace_authorization {
|
||||
return authorization
|
||||
.issue(
|
||||
method,
|
||||
path_and_query,
|
||||
workspace_runtime_operation(method, path_and_query),
|
||||
None,
|
||||
body,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let authorization = self.clone();
|
||||
Box::pin(async move {
|
||||
if let Some(authorizer) = &authorization.workspace_authorization {
|
||||
let operation = workspace_runtime_operation(&method, &path_and_query).to_string();
|
||||
return authorizer
|
||||
.issue_async(method, path_and_query, operation, None, body)
|
||||
.await
|
||||
.map_err(|error| WorkdirError::Unavailable(error.message));
|
||||
}
|
||||
authorization.fallback_bearer_token.ok_or_else(|| {
|
||||
WorkdirError::Unavailable(
|
||||
"remote Runtime does not have bearer authorization configured".to_string(),
|
||||
)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.message));
|
||||
}
|
||||
self.fallback_bearer_token.clone().ok_or_else(|| {
|
||||
WorkdirError::Unavailable(
|
||||
"remote Runtime does not have bearer authorization configured".to_string(),
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -6348,6 +6382,79 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() {
|
||||
use crate::db_executor::{DbExecutor, DbExecutorLimits};
|
||||
use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding};
|
||||
use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore;
|
||||
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::in_memory_for_test()
|
||||
.unwrap()
|
||||
.with_db_executor(executor.clone()),
|
||||
);
|
||||
let service = WorkspaceSigningIdentityService::new(
|
||||
store.clone(),
|
||||
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
|
||||
);
|
||||
let authorization = RemoteWorkdirAuthorization {
|
||||
workspace_authorization: Some(WorkspaceRuntimeAuthorization::new(
|
||||
store,
|
||||
service,
|
||||
"https://backend.test",
|
||||
WorkspaceRuntimeBinding {
|
||||
workspace_id: "workspace".into(),
|
||||
runtime_id: "runtime".into(),
|
||||
display_name: "Runtime".into(),
|
||||
base_url: "https://runtime.test".into(),
|
||||
public_key_fingerprint: "unused".into(),
|
||||
public_key: "unused".into(),
|
||||
binding_id: "binding".into(),
|
||||
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
revoked_at: None,
|
||||
},
|
||||
)),
|
||||
fallback_bearer_token: Some("must-not-be-used".into()),
|
||||
};
|
||||
|
||||
// Hold the running slot and fill the only queue slot. The callback must
|
||||
// reject admission, not run inline or silently switch authentication.
|
||||
let (release, wait) = std::sync::mpsc::channel::<()>();
|
||||
let (started, running) = tokio::sync::oneshot::channel();
|
||||
executor
|
||||
.dispatch(move || {
|
||||
let _ = started.send(());
|
||||
let _ = wait.recv_timeout(Duration::from_secs(5));
|
||||
})
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), running)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
executor.dispatch(|| {}).unwrap();
|
||||
let error = authorization
|
||||
.bearer_token(
|
||||
"POST".into(),
|
||||
"/v1/workdir-sessions/s/operations".into(),
|
||||
b"{}".to_vec(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string()
|
||||
));
|
||||
drop(release);
|
||||
executor.drain().await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ticket_worker_spawn_intent_uses_worker_wire_role() {
|
||||
let intent = WorkerSpawnIntent::TicketRole {
|
||||
|
||||
@@ -32,7 +32,8 @@ struct Fixture {
|
||||
impl Fixture {
|
||||
async fn new() -> Self {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap());
|
||||
let store =
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap());
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".into(),
|
||||
|
||||
@@ -45,7 +45,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() {
|
||||
let artifacts = evidence_rows(&conn, "artifacts");
|
||||
let receipts = evidence_rows(&conn, "repository_secret_legacy_receipts");
|
||||
drop(conn);
|
||||
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
upgraded
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
|
||||
@@ -73,7 +73,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() {
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
for store in [upgraded, SqliteWorkspaceStore::in_memory().unwrap()] {
|
||||
for store in [upgraded, SqliteWorkspaceStore::in_memory_for_test().unwrap()] {
|
||||
store.with_conn(|conn| {
|
||||
let mut stmt = conn.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")?;
|
||||
let objects = stmt.query_map([],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
|
||||
@@ -10,6 +10,7 @@ pub mod backend_job;
|
||||
pub mod companion;
|
||||
pub mod config;
|
||||
pub mod config_source;
|
||||
pub mod db_executor;
|
||||
pub mod feature_storage;
|
||||
pub mod hosts;
|
||||
pub mod memory_backend;
|
||||
@@ -159,6 +160,12 @@ pub enum Error {
|
||||
WorkspaceSigningIdentity { code: String, message: String },
|
||||
#[error("Workspace Drive error: {0}")]
|
||||
Drive(#[from] workspace_drive::Error),
|
||||
#[error("DB execution capacity exceeded")]
|
||||
DbCapacityExceeded,
|
||||
#[error("DB executor unavailable")]
|
||||
DbExecutorUnavailable,
|
||||
#[error("DB operation outcome unknown; do not automatically retry")]
|
||||
DbOperationOutcomeUnknown,
|
||||
#[error("store error: {0}")]
|
||||
Store(String),
|
||||
}
|
||||
|
||||
@@ -217,7 +217,7 @@ mod tests {
|
||||
async fn authority() -> (TempDir, SqliteWorkspaceAuthority) {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let db_path = temp.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-test".to_string(),
|
||||
|
||||
@@ -32,7 +32,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() {
|
||||
drop(conn);
|
||||
for attempt in 0..3 {
|
||||
let error = match attempt {
|
||||
0 => SqliteWorkspaceStore::open(&path).err().unwrap(),
|
||||
0 => SqliteWorkspaceStore::open_for_test(&path).err().unwrap(),
|
||||
1 => SqliteWorkspaceStore::migration_plan(&path).unwrap_err(),
|
||||
_ => SqliteWorkspaceStore::migrate_database(&path).unwrap_err(),
|
||||
};
|
||||
@@ -62,7 +62,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() {
|
||||
)
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
|
||||
assert_eq!(conn.query_row("SELECT input_fingerprint FROM worker_removal_operations WHERE operation_id='removal'",[],|r|r.get::<_,String>(0))?,"legacy-receipt");
|
||||
@@ -109,7 +109,7 @@ fn migration_dry_run_and_apply_share_the_original_repository_secret_authority()
|
||||
assert!(!table_exists(&conn, "server_secret_objects").unwrap());
|
||||
drop(conn);
|
||||
assert_eq!(SqliteWorkspaceStore::migrate_database(&path).unwrap(), plan);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?,LATEST_SCHEMA_VERSION);
|
||||
let (nonce, mut sealed): (Vec<u8>,Vec<u8>) = conn.query_row(
|
||||
|
||||
@@ -2290,7 +2290,7 @@ mod tests {
|
||||
) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let database = dir.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
|
||||
let timestamp = now();
|
||||
for workspace_id in ["workspace-a", "workspace-b"] {
|
||||
futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord {
|
||||
|
||||
@@ -1116,7 +1116,7 @@ mod tests {
|
||||
WorkerId::from_legacy_u64(1)
|
||||
}
|
||||
fn setup() -> SqliteWorkspaceStore {
|
||||
let s = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let s = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
s.with_conn(|c| {
|
||||
c.execute(
|
||||
"INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) \
|
||||
|
||||
@@ -965,7 +965,7 @@ async fn connect_runtime(
|
||||
let mut request = endpoint
|
||||
.into_client_request()
|
||||
.map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?;
|
||||
if let Some(token) = runtime_token(config, workspace_id)? {
|
||||
if let Some(token) = runtime_token(config, workspace_id).await? {
|
||||
request.headers_mut().insert(
|
||||
"authorization",
|
||||
format!("Bearer {token}")
|
||||
@@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String {
|
||||
format!("{base}/v1/protocol/ws")
|
||||
}
|
||||
}
|
||||
fn runtime_token(
|
||||
async fn runtime_token(
|
||||
config: &RemoteRuntimeConfig,
|
||||
_workspace_id: &str,
|
||||
) -> Result<Option<String>, String> {
|
||||
if let Some(authorization) = config.workspace_authorization.as_ref() {
|
||||
return authorization
|
||||
.issue("GET", "/v1/protocol/ws", "workers:list", None, &[])
|
||||
.issue_async(
|
||||
"GET".to_string(),
|
||||
"/v1/protocol/ws".to_string(),
|
||||
"workers:list".to_string(),
|
||||
None,
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(|error| error.message);
|
||||
}
|
||||
|
||||
@@ -38,8 +38,8 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("upgraded.db");
|
||||
prepare_retained_schema(&path, 85);
|
||||
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
let mut mismatches = Vec::new();
|
||||
for table in [
|
||||
"workspace_tags",
|
||||
|
||||
+5304
-3302
File diff suppressed because it is too large
Load Diff
@@ -65,70 +65,85 @@ pub(super) fn require_candidate_grant(
|
||||
/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface
|
||||
/// background work; the new Job regenerates it and does not infer surface success.
|
||||
pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult<bool> {
|
||||
let job_api = api.clone();
|
||||
let subject_id = subject_id.to_owned();
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let Some(owner) = api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
else {
|
||||
return Ok(true);
|
||||
let (worker, complete) = api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
let api = &job_api;
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let Some(owner) = api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
else {
|
||||
return Ok((None, true));
|
||||
};
|
||||
let mut summary = api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?;
|
||||
summary.singleton_key = Some(owner.key.clone());
|
||||
require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?;
|
||||
let stopped = summary.state.eq_ignore_ascii_case("stopped");
|
||||
let idle = summary.state.eq_ignore_ascii_case("idle")
|
||||
&& summary
|
||||
.worker_state
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
|
||||
if !stopped && !idle {
|
||||
return Ok((None, false));
|
||||
}
|
||||
if api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.as_ref()
|
||||
.map(|o| &o.worker)
|
||||
!= Some(&owner.worker)
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"legacy consolidation owner moved during cutover".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if !summary.state.eq_ignore_ascii_case("stopped") {
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(
|
||||
&owner.worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(
|
||||
"Retire legacy subjektiv consolidator after foreground completion"
|
||||
.into(),
|
||||
),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok((None, false));
|
||||
}
|
||||
}
|
||||
if !api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?
|
||||
.state
|
||||
.eq_ignore_ascii_case("stopped")
|
||||
{
|
||||
return Ok((None, false));
|
||||
}
|
||||
Ok::<_, ApiError>((Some(owner.worker), false))
|
||||
})
|
||||
.await?;
|
||||
let Some(worker) = worker else {
|
||||
return Ok(complete);
|
||||
};
|
||||
let mut summary = api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?;
|
||||
summary.singleton_key = Some(owner.key.clone());
|
||||
require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?;
|
||||
let stopped = summary.state.eq_ignore_ascii_case("stopped");
|
||||
let idle = summary.state.eq_ignore_ascii_case("idle")
|
||||
&& summary
|
||||
.worker_state
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
|
||||
if !stopped && !idle {
|
||||
return Ok(false);
|
||||
}
|
||||
if api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.as_ref()
|
||||
.map(|o| &o.worker)
|
||||
!= Some(&owner.worker)
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"legacy consolidation owner moved during cutover".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if !summary.state.eq_ignore_ascii_case("stopped") {
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(
|
||||
&owner.worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(
|
||||
"Retire legacy subjektiv consolidator after foreground completion".into(),
|
||||
),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
if !api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?
|
||||
.state
|
||||
.eq_ignore_ascii_case("stopped")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let removal = WorkerRemovalService::new(api)
|
||||
.execute_target_removal(
|
||||
api.runtime.as_ref(),
|
||||
&owner.worker,
|
||||
&worker,
|
||||
"subjektiv consolidation Job cutover",
|
||||
None,
|
||||
)
|
||||
@@ -141,10 +156,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
|
||||
))
|
||||
.into());
|
||||
}
|
||||
Ok(api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.is_none())
|
||||
let store = api.store.clone();
|
||||
let workspace_id = api.workspace_id().to_owned();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store
|
||||
.current_worker_singleton_owner(&workspace_id, &key)
|
||||
.map(|owner| owner.is_none())
|
||||
.map_err(ApiError::from)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn bounded_candidate_batch(
|
||||
|
||||
@@ -806,7 +806,19 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc
|
||||
.unwrap();
|
||||
};
|
||||
let ((status, body, events), ()) = tokio::join!(
|
||||
request_with_logs(build_router(fixture.api.clone()), request),
|
||||
async {
|
||||
let result = request_with_logs(build_router(fixture.api.clone()), request).await;
|
||||
// A pre-dispatch rejection leaves the fake provider waiting on an
|
||||
// open channel. Fail with the actual response here instead of
|
||||
// hanging join! until the outer suite timeout hides that cause.
|
||||
assert_eq!(
|
||||
result.0,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"early command-start response: {}",
|
||||
result.1
|
||||
);
|
||||
result
|
||||
},
|
||||
provider
|
||||
);
|
||||
assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}");
|
||||
@@ -832,3 +844,78 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc
|
||||
"post-dispatch failure was retried"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_host_before_command_start_returns_availability_without_a_provider_operation() {
|
||||
let mut fixture = manual_worker_assignment_fixture().await;
|
||||
let identity =
|
||||
worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id)
|
||||
.unwrap();
|
||||
configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id);
|
||||
let (sender, mut commands) = tokio::sync::mpsc::channel(1);
|
||||
let connection = Arc::new(ExternalProviderConnection {
|
||||
grant_id: "overload-grant".into(),
|
||||
workdir_id: fixture.main_workdir_id.clone(),
|
||||
provider_instance_id: "overload-provider".into(),
|
||||
generation: 1,
|
||||
expires_at: None,
|
||||
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||
admission: Arc::new(tokio::sync::Semaphore::new(1)),
|
||||
shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
sender,
|
||||
});
|
||||
fixture.runtime.workdir_session_factory = Some(Arc::new(move |_| {
|
||||
Ok(Arc::new(ExternalProviderWorkdirSession::new(
|
||||
connection.clone(),
|
||||
None,
|
||||
)) as WorkdirSessionHandle)
|
||||
}));
|
||||
fixture
|
||||
.api
|
||||
.runtime
|
||||
.register_or_replace(fixture.runtime.clone());
|
||||
let host = fixture.api.host_executors.host().unwrap();
|
||||
host.drain().await.unwrap();
|
||||
let mut releases = Vec::new(); // dropping senders also releases jobs on panic
|
||||
for _ in 0..2 {
|
||||
let (release, permit) = std::sync::mpsc::channel::<()>();
|
||||
releases.push(release);
|
||||
let (started, accepted) = tokio::sync::oneshot::channel();
|
||||
host.dispatch(move || {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv_timeout(std::time::Duration::from_secs(10));
|
||||
})
|
||||
.unwrap();
|
||||
accepted.await.unwrap();
|
||||
}
|
||||
for _ in 0..16 {
|
||||
host.dispatch(|| {}).unwrap();
|
||||
}
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations");
|
||||
let request = runtime_source_request(
|
||||
&identity,
|
||||
Some(&fixture.worker.worker_id),
|
||||
"POST",
|
||||
&path,
|
||||
serde_json::to_vec(&json!({
|
||||
"target_workdir": "checkout",
|
||||
"operation": {"operation": "command_start", "request": {
|
||||
"command": "not-dispatched", "timeout_secs": 1, "output_limit": 1024,
|
||||
"cwd": "", "spill_dir": null, "tool_call_id": "not-dispatched"
|
||||
}}
|
||||
}))
|
||||
.unwrap(),
|
||||
);
|
||||
let (status, body, _) = request_with_logs(build_router(fixture.api.clone()), request).await;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");
|
||||
assert!(body.to_string().contains("db_capacity_exceeded"), "{body}");
|
||||
// The response is already final but the live fake-provider receiver remains
|
||||
// Pending. An unbounded join of those two futures hid this causal rejection
|
||||
// behind the original 600-second suite timeout. No process or command ran.
|
||||
let provider_command = commands.recv();
|
||||
tokio::pin!(provider_command);
|
||||
assert!(futures::poll!(provider_command.as_mut()).is_pending());
|
||||
drop(releases);
|
||||
host.drain().await.unwrap();
|
||||
fixture.api.store.db_executor().drain().await.unwrap();
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
|
||||
configure_runtime_request_auth(&mut api, &identity, "runtime-test");
|
||||
let revoked = RuntimeIdentityMaterial::generate("revoked-runtime-secret").unwrap();
|
||||
configure_runtime_request_auth(&mut api, &revoked, "revoked-runtime-secret");
|
||||
SqliteWorkspaceStore::open(&api.config.database_path)
|
||||
SqliteWorkspaceStore::open_for_test(&api.config.database_path)
|
||||
.unwrap()
|
||||
.revoke_workspace_runtime_binding(
|
||||
TEST_WORKSPACE_ID,
|
||||
@@ -70,7 +70,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
|
||||
let body = "body-secret";
|
||||
let apps = [
|
||||
build_router(api.clone()),
|
||||
workspace_server_router(WorkspaceServerApi::new(
|
||||
workspace_server_router(WorkspaceServerApi::new_for_test(
|
||||
api.config.clone(),
|
||||
api.store.clone(),
|
||||
)),
|
||||
@@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m
|
||||
&error,
|
||||
)
|
||||
});
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
@@ -347,7 +347,7 @@ async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() {
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/repositories");
|
||||
for app in [
|
||||
build_router(api.clone()),
|
||||
workspace_server_router(WorkspaceServerApi::new(
|
||||
workspace_server_router(WorkspaceServerApi::new_for_test(
|
||||
api.config.clone(),
|
||||
api.store.clone(),
|
||||
)),
|
||||
|
||||
@@ -40,7 +40,7 @@ async fn cached_drive_handle_rejects_mutation_after_deletion_reservation() {
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let api = WorkspaceApi::new(config.clone(), store.clone())
|
||||
let api = WorkspaceApi::new_for_test(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let cached = api.drive().await.unwrap();
|
||||
@@ -104,7 +104,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature
|
||||
// Use a separate embedded-runtime namespace for the first API: this test
|
||||
// reconstructs Drive authority in-process rather than spawning/restarting
|
||||
// product processes (runtime observers may retain their namespace).
|
||||
let first_api = WorkspaceApi::new(config.clone(), store.clone())
|
||||
let first_api = WorkspaceApi::new_for_test(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
// Merely constructing a Workspace API must not create blob storage.
|
||||
@@ -160,7 +160,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
drop(first_api);
|
||||
let recovered = WorkspaceServerApi::new(config.clone(), store.clone());
|
||||
let recovered = WorkspaceServerApi::new_for_test(config.clone(), store.clone());
|
||||
let api = recovered
|
||||
.api_for_workspace(&config.workspace_id)
|
||||
.await
|
||||
|
||||
@@ -39,10 +39,10 @@ impl RealDriveFixture {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(temp.path());
|
||||
config.repositories.clear();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
let owner = seed_test_api_token(store.as_ref(), "drive-web-owner");
|
||||
let second = seed_test_api_token(store.as_ref(), "drive-web-second");
|
||||
let server_api = WorkspaceServerApi::new(config, store.clone());
|
||||
let server_api = WorkspaceServerApi::new_for_test(config, store.clone());
|
||||
let server = LoopbackServer::start(workspace_server_router(server_api.clone())).await;
|
||||
// Exercise T-717's real repositoryless Workspace bootstrap, not a
|
||||
// pre-existing test Workspace with its repository merely hidden in UI.
|
||||
|
||||
@@ -767,6 +767,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) {
|
||||
std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 })
|
||||
);
|
||||
assert!(execution.protocol_methods().is_empty());
|
||||
// SQLite jobs run on external threads; paused Tokio time cannot observe their
|
||||
// progress and may auto-advance a reconnect timeout before SQL completes.
|
||||
tokio::time::resume();
|
||||
assert_same_worker_reconnect_and_stop(&worker, &context).await;
|
||||
}
|
||||
|
||||
@@ -844,7 +847,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[tokio::test]
|
||||
async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress()
|
||||
{
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -27,7 +27,7 @@ async fn subject_session_callback_completes_while_create_and_deletion_are_waitin
|
||||
)
|
||||
.unwrap();
|
||||
let worker_id = reserved.worker_id.to_string();
|
||||
let server = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
|
||||
let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
|
||||
// Reuse the fixture API, but exercise the real outer dispatcher and both auth layers.
|
||||
server
|
||||
.apis
|
||||
|
||||
@@ -144,7 +144,7 @@ impl LaunchFixture {
|
||||
async fn new(remote: bool) -> Self {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let config = test_server_config(temp.path());
|
||||
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap();
|
||||
let backend = Arc::new(LaunchExecutionBackend::default());
|
||||
let api =
|
||||
WorkspaceApi::new_with_execution_backend(config, Arc::new(store), backend.clone())
|
||||
|
||||
@@ -151,7 +151,7 @@ impl Fixture {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(temp.path());
|
||||
config.repositories.clear();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
let mut api = WorkspaceApi::new_with_execution_backend(
|
||||
config,
|
||||
store,
|
||||
|
||||
@@ -830,7 +830,11 @@ async fn grant_replacement_while_admission_waits_cannot_use_another_grants_lock(
|
||||
};
|
||||
let admission = handle.set_pinned(&context, true);
|
||||
tokio::pin!(admission);
|
||||
// Poll deterministically to the held lock before replacing grant authority.
|
||||
// The first poll can now be waiting for the offloaded authority read, not
|
||||
// the held grant lock. Drain this fixture's actual SQL, then poll its saved
|
||||
// reply to reach admission before replacing authority.
|
||||
assert!(futures::poll!(admission.as_mut()).is_pending());
|
||||
api.store.db_executor().drain().await.unwrap();
|
||||
assert!(futures::poll!(admission.as_mut()).is_pending());
|
||||
api.store
|
||||
.revoke_worker_control_grant(
|
||||
@@ -1329,8 +1333,9 @@ async fn restore_guard_internal_stale_conflict_retires_intent_for_later_explicit
|
||||
);
|
||||
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||
// Retirement, unlike in-memory refresh, must survive caller journal restart.
|
||||
api.config_store =
|
||||
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
|
||||
api.config_store = Arc::new(
|
||||
SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(),
|
||||
);
|
||||
let fresh = api.runtime.worker(&identity).unwrap();
|
||||
let next = api
|
||||
.config_store
|
||||
@@ -1490,7 +1495,7 @@ async fn restore_guard_internal_intent_survives_client_store_restart_without_rep
|
||||
);
|
||||
// Reopen the durable caller journal, discarding all client-side memory of request identity.
|
||||
api.config_store =
|
||||
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap());
|
||||
let pinned = api
|
||||
.config_store
|
||||
.pin_internal_worker_restore_intent(
|
||||
@@ -1599,7 +1604,7 @@ async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_w
|
||||
crate::repository_source::repository_source_fingerprint(&source);
|
||||
config.repositories[0].source = source;
|
||||
config.repositories[0].path = None;
|
||||
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap();
|
||||
let execution = Arc::new(DeterministicExecutionBackend::default());
|
||||
execution.accept_restores();
|
||||
let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone())
|
||||
|
||||
@@ -56,7 +56,7 @@ impl Fixture {
|
||||
api.runtime.register_or_replace(runtime);
|
||||
let now = Arc::new(AtomicI64::new(START));
|
||||
let clock = now.clone();
|
||||
let mut server = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
|
||||
let mut server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
|
||||
server.proof_clock = Arc::new(move || clock.load(Ordering::SeqCst));
|
||||
server
|
||||
.apis
|
||||
@@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
|
||||
let f = Fixture::new().await;
|
||||
let mut a = f.attach("1");
|
||||
let (first, reply) = f.start_read("1", &mut a).await;
|
||||
let mut second = Box::pin(f.app.clone().oneshot(f.operation("1")));
|
||||
// Poll through dispatch/auth until the handler waits on its resource lock.
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let request = f.operation("1");
|
||||
let proof = request
|
||||
.headers()
|
||||
.get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER)
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap();
|
||||
let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof)
|
||||
.unwrap()
|
||||
.jti;
|
||||
let mut second = Box::pin(f.app.clone().oneshot(request));
|
||||
// DB auth is now genuinely asynchronous. Observe durable proof consumption,
|
||||
// not a guessed number of polls, before advancing the injected proof clock.
|
||||
let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
tokio::time::timeout(BOUND, async {
|
||||
loop {
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let consumed: bool = connection
|
||||
.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)",
|
||||
[&jti],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
if consumed {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second request never authenticated before resource wait");
|
||||
assert!(
|
||||
a.try_recv().is_err(),
|
||||
"same session read escaped resource exclusion"
|
||||
@@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
|
||||
assert_json(first.await.unwrap(), StatusCode::OK).await;
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let Some(ExternalProviderCommand::Operation { response, .. }) =
|
||||
tokio::time::timeout(BOUND, a.recv()).await.unwrap()
|
||||
tokio::time::timeout(BOUND, async {
|
||||
tokio::select! {
|
||||
command = a.recv() => command,
|
||||
result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()),
|
||||
}
|
||||
}).await.unwrap()
|
||||
else {
|
||||
panic!("expected second read")
|
||||
};
|
||||
@@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits(
|
||||
}),
|
||||
);
|
||||
let mut deletion = Box::pin(f.app.clone().oneshot(request));
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await;
|
||||
assert!(
|
||||
gate.admit(false).is_none(),
|
||||
"deletion start did not fence admission"
|
||||
);
|
||||
tokio::time::timeout(BOUND, async {
|
||||
loop {
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
if gate.admit(false).is_none() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("deletion start did not fence admission after async authentication");
|
||||
let session_id = session_store::new_session_id().to_string();
|
||||
let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions");
|
||||
let callback = f.request(
|
||||
|
||||
@@ -57,6 +57,19 @@ impl WorkerOperationContext {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) async fn from_request_async(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
) -> std::result::Result<Self, server_api::RepositoryApiError> {
|
||||
let owned_api = api.clone();
|
||||
let context = context.clone();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context)))
|
||||
.await
|
||||
.map_err(|error| ApiError::from(error).into_repository_api_error())?
|
||||
}
|
||||
|
||||
pub(super) fn from_request(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
@@ -162,8 +175,20 @@ impl<Services> WorkspaceWorker<Services> {
|
||||
source: crate::worker_source::VerifiedWorkerMutationSource,
|
||||
reason: &str,
|
||||
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
||||
self.current_record()
|
||||
.map_err(|error| error.error.to_string())?;
|
||||
let removal = self.removal.clone();
|
||||
let identity = self.identity.clone();
|
||||
self.removal
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
removal
|
||||
.store
|
||||
.get_worker_registry(&removal.workspace_id, &identity)?
|
||||
.ok_or_else(|| Error::UnknownWorker { worker: identity })
|
||||
.map(drop)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
self.removal
|
||||
.execute_async(
|
||||
source,
|
||||
@@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor
|
||||
}
|
||||
|
||||
impl WorkspaceWorker {
|
||||
pub(crate) async fn resolve_async(
|
||||
api: &WorkspaceApi,
|
||||
runtime_id: &str,
|
||||
reference: &str,
|
||||
) -> Result<Self> {
|
||||
let owned_api = api.clone();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let reference = reference.to_owned();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || Self::resolve(&owned_api, &runtime_id, &reference))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result<Self> {
|
||||
Self::resolve_with_services(
|
||||
api.clone(),
|
||||
@@ -239,7 +278,36 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
permission: &str,
|
||||
) -> ApiResult<Option<tokio::sync::OwnedMutexGuard<()>>> {
|
||||
self.current_record()?;
|
||||
let worker = self.clone();
|
||||
let initial_context = context.clone();
|
||||
let permission = permission.to_owned();
|
||||
let initial_permission = permission.clone();
|
||||
let grant = self
|
||||
.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<Option<WorkerControlGrantRecord>> {
|
||||
worker.current_record()?;
|
||||
let WorkerOperationContext::WorkerControl { controller } = initial_context else {
|
||||
return Ok(None);
|
||||
};
|
||||
if worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(worker.api.workspace_id(), &controller)?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::UnknownWorker { worker: controller }.into());
|
||||
}
|
||||
Ok(Some(authorize_known_worker_permission(
|
||||
&worker.api,
|
||||
worker.api.workspace_id(),
|
||||
&controller,
|
||||
&worker.identity,
|
||||
&initial_permission,
|
||||
)?))
|
||||
})
|
||||
.await?;
|
||||
match context {
|
||||
#[cfg(test)]
|
||||
WorkerOperationContext::Backend => Ok(None),
|
||||
@@ -248,42 +316,35 @@ impl WorkspaceWorker {
|
||||
Ok(None)
|
||||
}
|
||||
WorkerOperationContext::WorkerControl { controller } => {
|
||||
if self
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(self.api.workspace_id(), controller)?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::UnknownWorker {
|
||||
worker: controller.clone(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let grant = authorize_known_worker_permission(
|
||||
&self.api,
|
||||
self.api.workspace_id(),
|
||||
controller,
|
||||
&self.identity,
|
||||
permission,
|
||||
)?;
|
||||
let grant = grant.expect("Worker control context resolved a grant");
|
||||
// Await admission outside the DB lane; the recheck owns the guard
|
||||
// until it finishes, even if its async waiter is cancelled.
|
||||
let guard = worker_control_lock(&self.api, &grant.grant_id)
|
||||
.lock_owned()
|
||||
.await;
|
||||
self.current_record()?;
|
||||
let current_grant = authorize_known_worker_permission(
|
||||
&self.api,
|
||||
self.api.workspace_id(),
|
||||
controller,
|
||||
&self.identity,
|
||||
permission,
|
||||
)?;
|
||||
if current_grant.grant_id != grant.grant_id {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Worker control grant changed while waiting for admission".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(Some(guard))
|
||||
let worker = self.clone();
|
||||
let controller = controller.clone();
|
||||
self.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<_> {
|
||||
worker.current_record()?;
|
||||
let current_grant = authorize_known_worker_permission(
|
||||
&worker.api,
|
||||
worker.api.workspace_id(),
|
||||
&controller,
|
||||
&worker.identity,
|
||||
&permission,
|
||||
)?;
|
||||
if current_grant.grant_id != grant.grant_id {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Worker control grant changed while waiting for admission".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(Some(guard))
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -293,8 +354,16 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
) -> ApiResult<InternalWorkerRestoreResult> {
|
||||
let _guard = self.authorize_operation(context, "restore").await?;
|
||||
self.restore_internal(request)
|
||||
let guard = self.authorize_operation(context, "restore").await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
worker.restore_internal(request)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Trusted Backend callers still supply an explicit observed intent. Runtime
|
||||
@@ -312,27 +381,48 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: WorkerLifecycleRequest,
|
||||
) -> ApiResult<WorkerLifecycleResult> {
|
||||
let _guard = self.authorize_operation(context, "stop").await?;
|
||||
let result = self
|
||||
let guard = self.authorize_operation(context, "stop").await?;
|
||||
let worker = self.clone();
|
||||
let (result, guard) = self
|
||||
.api
|
||||
.runtime
|
||||
.stop_worker(&self.identity, request)
|
||||
.map_err(|e| e.into_error())?;
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let result = worker
|
||||
.api
|
||||
.runtime
|
||||
.stop_worker(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?;
|
||||
Ok((result, guard))
|
||||
})
|
||||
.await?;
|
||||
if result.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(result);
|
||||
}
|
||||
parse_runtime_worker_id_for_registry(&self.identity.worker_id)?;
|
||||
let session_lock = current_worker_session_lock(&self.api, &self.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
let session_guard = session_lock.lock_owned().await;
|
||||
close_current_worker_session_locked(&self.api, &self.identity).await?;
|
||||
if let Some(record) = self
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(self.api.workspace_id(), &self.identity)?
|
||||
{
|
||||
sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?;
|
||||
}
|
||||
Ok(result)
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let (_control, _session) = (guard, session_guard);
|
||||
if let Some(record) = worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(worker.api.workspace_id(), &worker.identity)?
|
||||
{
|
||||
sync_linked_workdir_after_worker_stop(
|
||||
&worker.api,
|
||||
&worker.identity.runtime_id,
|
||||
&record,
|
||||
)?;
|
||||
}
|
||||
Ok(result)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn cancel(
|
||||
@@ -340,12 +430,20 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: WorkerLifecycleRequest,
|
||||
) -> ApiResult<WorkerLifecycleResult> {
|
||||
let _guard = self.authorize_operation(context, "cancel").await?;
|
||||
Ok(self
|
||||
.api
|
||||
.runtime
|
||||
.cancel_worker(&self.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
let guard = self.authorize_operation(context, "cancel").await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let _guard = guard;
|
||||
Ok(worker
|
||||
.api
|
||||
.runtime
|
||||
.cancel_worker(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn input(
|
||||
@@ -358,46 +456,61 @@ impl WorkspaceWorker {
|
||||
} else {
|
||||
"send_input"
|
||||
};
|
||||
let _guard = self.authorize_operation(context, permission).await?;
|
||||
let guard = self.authorize_operation(context, permission).await?;
|
||||
let session_lock = current_worker_session_lock(&self.api, &self.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
self.ensure_interactive()?;
|
||||
let session_guard = session_lock.lock_owned().await;
|
||||
self.ensure_interactive().await?;
|
||||
if matches!(context, WorkerOperationContext::WorkerControl { .. })
|
||||
&& request.kind == WorkerInputKind::User
|
||||
{
|
||||
request.kind = WorkerInputKind::UserIfIdle;
|
||||
}
|
||||
Ok(self
|
||||
.api
|
||||
.runtime
|
||||
.send_input(&self.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let (_control, _session) = (guard, session_guard);
|
||||
Ok(worker
|
||||
.api
|
||||
.runtime
|
||||
.send_input(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn ensure_interactive(&self) -> ApiResult<()> {
|
||||
if let Some(claim) = self
|
||||
.api
|
||||
async fn ensure_interactive(&self) -> ApiResult<()> {
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.store
|
||||
.get_pending_ticket_assignment_operation_for_worker(
|
||||
self.api.workspace_id(),
|
||||
&self.identity,
|
||||
)?
|
||||
{
|
||||
return Err(Error::TicketAssignmentConflict(format!(
|
||||
"Worker has pending Ticket claim {}; reconcile that exact operation before admitting input",
|
||||
claim.operation_id,
|
||||
)).into());
|
||||
}
|
||||
if self
|
||||
.api
|
||||
.store
|
||||
.worker_registry_projection(self.api.workspace_id(), &self.identity)?
|
||||
.and_then(|projection| projection.job)
|
||||
.is_some()
|
||||
{
|
||||
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
|
||||
}
|
||||
Ok(())
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<()> {
|
||||
if let Some(claim) = worker
|
||||
.api
|
||||
.store
|
||||
.get_pending_ticket_assignment_operation_for_worker(
|
||||
worker.api.workspace_id(),
|
||||
&worker.identity,
|
||||
)?
|
||||
{
|
||||
return Err(Error::TicketAssignmentConflict(format!(
|
||||
"Worker has pending Ticket claim {}; reconcile that exact operation before admitting input",
|
||||
claim.operation_id,
|
||||
)).into());
|
||||
}
|
||||
if worker
|
||||
.api
|
||||
.store
|
||||
.worker_registry_projection(worker.api.workspace_id(), &worker.identity)?
|
||||
.and_then(|projection| projection.job)
|
||||
.is_some()
|
||||
{
|
||||
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn set_pinned(
|
||||
@@ -405,33 +518,44 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
pinned: bool,
|
||||
) -> ApiResult<WorkerRetentionResponse> {
|
||||
let _guard = self.authorize_operation(context, "pin").await?;
|
||||
let retention_state = if pinned { "pinned" } else { "normal" };
|
||||
let changed = self.api.worker_projection.publish_ordered(|| {
|
||||
self.api
|
||||
.store
|
||||
.update_worker_retention(
|
||||
self.api.workspace_id(),
|
||||
&self.identity,
|
||||
retention_state,
|
||||
now_registry_timestamp().as_str(),
|
||||
)
|
||||
.map(|commit| (!commit.changes.is_empty(), commit))
|
||||
})?;
|
||||
if !changed {
|
||||
return Err(cleanup_api_error(
|
||||
&self.identity.runtime_id,
|
||||
"workspace_worker_retention_unknown_worker",
|
||||
"Worker is not known to the Backend registry",
|
||||
));
|
||||
}
|
||||
Ok(WorkerRetentionResponse {
|
||||
workspace_id: self.api.workspace_id().to_owned(),
|
||||
runtime_id: self.identity.runtime_id.clone(),
|
||||
worker_id: self.identity.worker_id.clone(),
|
||||
pinned,
|
||||
retention_state: retention_state.into(),
|
||||
})
|
||||
let guard = self.authorize_operation(context, "pin").await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<WorkerRetentionResponse> {
|
||||
// Publication and its DB mutation are one owned stage. Accepted
|
||||
// work retains control admission through commit and publication.
|
||||
let _guard = guard;
|
||||
let retention_state = if pinned { "pinned" } else { "normal" };
|
||||
let changed = worker.api.worker_projection.publish_ordered(|| {
|
||||
worker
|
||||
.api
|
||||
.store
|
||||
.update_worker_retention(
|
||||
worker.api.workspace_id(),
|
||||
&worker.identity,
|
||||
retention_state,
|
||||
now_registry_timestamp().as_str(),
|
||||
)
|
||||
.map(|commit| (!commit.changes.is_empty(), commit))
|
||||
})?;
|
||||
if !changed {
|
||||
return Err(cleanup_api_error(
|
||||
&worker.identity.runtime_id,
|
||||
"workspace_worker_retention_unknown_worker",
|
||||
"Worker is not known to the Backend registry",
|
||||
));
|
||||
}
|
||||
Ok(WorkerRetentionResponse {
|
||||
workspace_id: worker.api.workspace_id().to_owned(),
|
||||
runtime_id: worker.identity.runtime_id.clone(),
|
||||
worker_id: worker.identity.worker_id.clone(),
|
||||
pinned,
|
||||
retention_state: retention_state.into(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// The caller owns the full cleanup-plan digest check. The removal service
|
||||
@@ -611,7 +735,7 @@ impl WorkspaceWorkerMethodSender {
|
||||
.await?;
|
||||
let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
self.worker.ensure_interactive()?;
|
||||
self.worker.ensure_interactive().await?;
|
||||
let WorkerOperationContext::Browser { source, .. } = context else {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"Browser Worker protocol context required".into(),
|
||||
|
||||
@@ -72,6 +72,28 @@ pub(super) struct RequestLease {
|
||||
callback: bool,
|
||||
}
|
||||
|
||||
impl crate::db_executor::DbOperationLifetime for RequestLease {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
// This is continuation of an already admitted request, not fresh admission.
|
||||
// Retain its drain responsibility before transferring work to the DB owner;
|
||||
// dropping/aborting the HTTP waiter must not allow deletion past live SQL.
|
||||
let mut state = self
|
||||
.gate
|
||||
.state
|
||||
.lock()
|
||||
.expect("Workspace admission poisoned");
|
||||
if self.callback {
|
||||
state.callbacks += 1;
|
||||
} else {
|
||||
state.requests += 1;
|
||||
}
|
||||
Box::new(RequestLease {
|
||||
gate: self.gate.clone(),
|
||||
callback: self.callback,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RequestLease {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self
|
||||
|
||||
@@ -597,7 +597,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let config = test_server_config(dir.path());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -769,7 +769,7 @@ async fn delivery_real_fixture(
|
||||
DeliveryScriptedClient,
|
||||
) {
|
||||
let config = test_server_config(root);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1007,7 +1007,7 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
||||
tokio::task::yield_now().await;
|
||||
let reopened = WorkspaceApi::new_with_execution_backend(
|
||||
config.clone(),
|
||||
Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()),
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()),
|
||||
execution.clone(),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,769 @@
|
||||
//! Real SQLite waits at the HTTP/authority boundary (not mock executor jobs).
|
||||
use super::*;
|
||||
use crate::SqliteWorkspaceStore;
|
||||
use crate::db_executor::{DbExecutor, DbExecutorLimits, DbOperationLifetime};
|
||||
use rusqlite::{Connection, ffi};
|
||||
use std::{
|
||||
ffi::c_void,
|
||||
future::Future,
|
||||
num::NonZeroUsize,
|
||||
sync::{atomic::AtomicBool, mpsc},
|
||||
thread::JoinHandle,
|
||||
time::Duration as StdDuration,
|
||||
};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
const WORKSPACE: &str = "db-execution";
|
||||
const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects";
|
||||
const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2);
|
||||
const WATCHDOG: StdDuration = StdDuration::from_secs(5);
|
||||
|
||||
async fn observed<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::time::timeout(OBSERVE_LIMIT, future)
|
||||
.await
|
||||
.expect("request did not reach the expected DB/lifecycle state")
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
// Keep temporary authority files alive until all other fixture owners drop.
|
||||
server: WorkspaceServerApi,
|
||||
api: WorkspaceApi,
|
||||
executor: DbExecutor,
|
||||
_dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
async fn new() -> Self {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
// Explicit startup config: never resolve the developer's live data dir.
|
||||
let config = ServerConfig {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
workspace_display_name: "DB execution fixture".into(),
|
||||
workspace_created_at: "2026-01-01T00:00:00Z".into(),
|
||||
workspace_execution_root: dir.path().into(),
|
||||
database_path: dir.path().join("server.db"),
|
||||
data_root: dir.path().join("data"),
|
||||
embedded_runtime_store_root: dir.path().join("runtime"),
|
||||
static_assets_dir: None,
|
||||
auth: AuthConfig::Passkey {
|
||||
rp_id: "localhost".into(),
|
||||
origin: "http://localhost:5173".into(),
|
||||
public_base_url: "http://localhost:5173".into(),
|
||||
cookie_name: "db_execution_session".into(),
|
||||
},
|
||||
max_records: 100,
|
||||
repositories: Vec::new(),
|
||||
runtime_event_sources: Vec::new(),
|
||||
remote_runtime_sources: Vec::new(),
|
||||
backend_base_url: Some("http://localhost:8787".into()),
|
||||
};
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open_for_test(&config.database_path)
|
||||
.unwrap()
|
||||
.with_db_executor(executor.clone()),
|
||||
);
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_user(&UserRecord {
|
||||
user_id: "fixture-user".into(),
|
||||
account_id: format!("account-{WORKSPACE}"),
|
||||
handle: "fixture-user".into(),
|
||||
display_name: "Fixture user".into(),
|
||||
created_at: config.workspace_created_at.clone(),
|
||||
updated_at: config.workspace_created_at.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.create_browser_session(&crate::store::BrowserSessionRecord {
|
||||
session_id: "fixture-session".into(),
|
||||
token_hash: crate::auth::token_hash("fixture-browser-token"),
|
||||
user_id: "fixture-user".into(),
|
||||
created_at: config.workspace_created_at.clone(),
|
||||
expires_at: "2999-01-01T00:00:00Z".into(),
|
||||
revoked_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let api = WorkspaceApi::new_for_test(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let server = WorkspaceServerApi::new_for_test(config, store);
|
||||
// Cache only the real inner router; avoid starting the unrelated hook.
|
||||
server
|
||||
.apis
|
||||
.lock()
|
||||
.await
|
||||
.insert(WORKSPACE.into(), api.clone());
|
||||
server
|
||||
.routers
|
||||
.lock()
|
||||
.await
|
||||
.insert(WORKSPACE.into(), build_inner_router(api.clone()));
|
||||
executor.drain().await.unwrap();
|
||||
Self {
|
||||
server,
|
||||
api,
|
||||
executor,
|
||||
_dir: dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn request(method: &str, path: &str, body: serde_json::Value) -> Request {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(path)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header("cookie", "db_execution_session=fixture-browser-token")
|
||||
.header(ORIGIN, "http://localhost:5173")
|
||||
.body(axum::body::Body::from(body.to_string()))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn subject_request(role: &str) -> Request {
|
||||
Self::request(
|
||||
"POST",
|
||||
SUBJECTS,
|
||||
serde_json::json!({ "role": role, "behavior_md": "" }),
|
||||
)
|
||||
}
|
||||
|
||||
fn dispatch(&self, request: Request) -> tokio::task::JoinHandle<Response> {
|
||||
let server = self.server.clone();
|
||||
tokio::spawn(async move { dispatch_workspace_request(State(server), request).await })
|
||||
}
|
||||
|
||||
// A pre-authenticated generated router isolates handler admission from the
|
||||
// earlier browser-session DB read. Full authentication/dispatch is exercised
|
||||
// separately by the cancellation and warm-Feature tests below.
|
||||
fn inner_router(&self) -> Router {
|
||||
build_inner_router(self.api.clone()).layer(Extension(RequestActor {
|
||||
user_id: "fixture-user".into(),
|
||||
account_id: format!("account-{WORKSPACE}"),
|
||||
handle: "fixture-user".into(),
|
||||
display_name: "Fixture user".into(),
|
||||
auth_method: ActorAuthMethod::BrowserSession,
|
||||
}))
|
||||
}
|
||||
|
||||
fn feature_database(&self) -> (crate::FeatureDatabase, PathBuf) {
|
||||
let database = self
|
||||
.api
|
||||
.feature_storage
|
||||
.open(&self.api.subjektiv_registration)
|
||||
.unwrap();
|
||||
// Initialize the trusted scope, then obtain SQLite's actual file path
|
||||
// from the managed handle rather than guessing the Feature directory.
|
||||
crate::subjektiv::SubjektivStore::from_database(database.clone()).unwrap();
|
||||
let path = database
|
||||
.with_connection(|conn| {
|
||||
Ok(PathBuf::from(conn.query_row(
|
||||
"SELECT file FROM pragma_database_list WHERE name = 'main'",
|
||||
[],
|
||||
|row| row.get::<_, String>(0),
|
||||
)?))
|
||||
})
|
||||
.unwrap();
|
||||
(database, path)
|
||||
}
|
||||
}
|
||||
|
||||
/// SQLite write ownership lives on an OS thread, including a bounded watchdog.
|
||||
/// Drop releases and joins it even when an assertion panics on the async side.
|
||||
struct WriteLock {
|
||||
release: Option<mpsc::Sender<()>>,
|
||||
thread: Option<JoinHandle<()>>,
|
||||
released: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WriteLock {
|
||||
fn acquire(path: PathBuf) -> Self {
|
||||
let (ready, held) = mpsc::sync_channel(1);
|
||||
let (release, permit) = mpsc::channel();
|
||||
let released = Arc::new(AtomicBool::new(false));
|
||||
let flag = released.clone();
|
||||
let thread = std::thread::spawn(move || {
|
||||
let conn = Connection::open(path).unwrap();
|
||||
conn.execute_batch("BEGIN IMMEDIATE").unwrap();
|
||||
ready.send(()).unwrap();
|
||||
let _ = permit.recv_timeout(WATCHDOG);
|
||||
conn.execute_batch("ROLLBACK").unwrap();
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
held.recv_timeout(WATCHDOG)
|
||||
.expect("SQLite lock holder did not acquire its transaction");
|
||||
Self {
|
||||
release: Some(release),
|
||||
thread: Some(thread),
|
||||
released,
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&mut self) {
|
||||
if let Some(release) = self.release.take() {
|
||||
let _ = release.send(());
|
||||
}
|
||||
if let Some(thread) = self.thread.take() {
|
||||
thread.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Drop for WriteLock {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
}
|
||||
}
|
||||
|
||||
struct BusyGate {
|
||||
entered: Option<oneshot::Sender<()>>,
|
||||
resume: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
/// rusqlite's fn-pointer busy_handler cannot capture a fixture. SQLite's
|
||||
/// userdata API can, without process globals or changing production hooks.
|
||||
enum BusyDatabase {
|
||||
Feature(crate::FeatureDatabase),
|
||||
Store(Arc<SqliteWorkspaceStore>),
|
||||
}
|
||||
impl BusyDatabase {
|
||||
fn access(&self, access: impl FnOnce(&Connection)) {
|
||||
match self {
|
||||
Self::Feature(database) => database
|
||||
.with_connection(|conn| {
|
||||
access(conn);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap(),
|
||||
Self::Store(store) => store
|
||||
.with_conn(|conn| {
|
||||
access(conn);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
struct BusyObservation {
|
||||
database: BusyDatabase,
|
||||
gate: Box<BusyGate>,
|
||||
resume: Option<mpsc::Sender<()>>,
|
||||
}
|
||||
impl BusyObservation {
|
||||
fn install(database: crate::FeatureDatabase) -> (Self, oneshot::Receiver<()>) {
|
||||
Self::install_on(BusyDatabase::Feature(database))
|
||||
}
|
||||
fn install_store(store: Arc<SqliteWorkspaceStore>) -> (Self, oneshot::Receiver<()>) {
|
||||
Self::install_on(BusyDatabase::Store(store))
|
||||
}
|
||||
fn install_on(database: BusyDatabase) -> (Self, oneshot::Receiver<()>) {
|
||||
unsafe extern "C" fn busy(context: *mut c_void, _: i32) -> i32 {
|
||||
// SAFETY: owned Box remains registered until Drop unregisters it,
|
||||
// serialized by the managed connection mutex.
|
||||
let gate = unsafe { &mut *context.cast::<BusyGate>() };
|
||||
if let Some(entered) = gate.entered.take() {
|
||||
let _ = entered.send(());
|
||||
return i32::from(gate.resume.recv_timeout(WATCHDOG).is_ok());
|
||||
}
|
||||
0
|
||||
}
|
||||
let (entered, waiting) = oneshot::channel();
|
||||
let (resume, permit) = mpsc::channel();
|
||||
let mut gate = Box::new(BusyGate {
|
||||
entered: Some(entered),
|
||||
resume: permit,
|
||||
});
|
||||
database.access(|conn| {
|
||||
// SAFETY: handle belongs to this connection; the Box is stable.
|
||||
let code = unsafe {
|
||||
ffi::sqlite3_busy_handler(
|
||||
conn.handle(),
|
||||
Some(busy),
|
||||
(&mut *gate as *mut BusyGate).cast(),
|
||||
)
|
||||
};
|
||||
assert_eq!(code, ffi::SQLITE_OK);
|
||||
});
|
||||
(
|
||||
Self {
|
||||
database,
|
||||
gate,
|
||||
resume: Some(resume),
|
||||
},
|
||||
waiting,
|
||||
)
|
||||
}
|
||||
fn resume(&mut self) {
|
||||
if let Some(resume) = self.resume.take() {
|
||||
let _ = resume.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Drop for BusyObservation {
|
||||
fn drop(&mut self) {
|
||||
self.resume();
|
||||
self.database.access(|conn| {
|
||||
// SAFETY: unregister before the boxed userdata is freed.
|
||||
unsafe {
|
||||
ffi::sqlite3_busy_handler(conn.handle(), None, std::ptr::null_mut());
|
||||
}
|
||||
});
|
||||
let _ = &self.gate;
|
||||
}
|
||||
}
|
||||
|
||||
// A handler with one DB stage can expose its admission without adding a test
|
||||
// hook to production. Polling its future to Pending proves run accepted it.
|
||||
struct AdmissionObservation(Mutex<Option<oneshot::Sender<()>>>);
|
||||
impl DbOperationLifetime for AdmissionObservation {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
if let Some(accepted) = self.0.lock().unwrap().take() {
|
||||
let _ = accepted.send(());
|
||||
}
|
||||
Box::new(())
|
||||
}
|
||||
}
|
||||
fn admission_observation() -> (Arc<dyn DbOperationLifetime>, oneshot::Receiver<()>) {
|
||||
let (accepted, receiver) = oneshot::channel();
|
||||
(
|
||||
Arc::new(AdmissionObservation(Mutex::new(Some(accepted)))),
|
||||
receiver,
|
||||
)
|
||||
}
|
||||
|
||||
struct StateCheckSaturation {
|
||||
executor: DbExecutor,
|
||||
stages: std::sync::atomic::AtomicUsize,
|
||||
releases: Arc<Mutex<Vec<mpsc::Sender<()>>>>,
|
||||
}
|
||||
impl DbOperationLifetime for StateCheckSaturation {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
if self.stages.fetch_add(1, Ordering::SeqCst) == 1 {
|
||||
// Authentication has completed; saturate just before the lifecycle
|
||||
// lookup is admitted. The callback's own dispatch admissions are
|
||||
// distinct stages and do not recurse into this branch.
|
||||
let (started, running) = mpsc::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
self.executor
|
||||
.dispatch(move || {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv_timeout(WATCHDOG);
|
||||
})
|
||||
.unwrap();
|
||||
running
|
||||
.recv_timeout(WATCHDOG)
|
||||
.expect("state-check blocker did not start");
|
||||
self.releases.lock().unwrap().push(release);
|
||||
let (release, permit) = mpsc::channel();
|
||||
self.executor
|
||||
.dispatch(move || {
|
||||
let _ = permit.recv_timeout(WATCHDOG);
|
||||
})
|
||||
.unwrap();
|
||||
self.releases.lock().unwrap().push(release);
|
||||
}
|
||||
Box::new(())
|
||||
}
|
||||
}
|
||||
struct ReleaseStateCheckBlockers(Arc<Mutex<Vec<mpsc::Sender<()>>>>);
|
||||
impl Drop for ReleaseStateCheckBlockers {
|
||||
fn drop(&mut self) {
|
||||
self.0.lock().unwrap().clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn saturated_mutation_lifecycle_lookup_returns_availability_not_deleting_conflict() {
|
||||
let fixture = Fixture::new().await;
|
||||
let releases = Arc::new(Mutex::new(Vec::new()));
|
||||
let release = ReleaseStateCheckBlockers(releases.clone());
|
||||
let source = Arc::new(StateCheckSaturation {
|
||||
executor: fixture.executor.clone(),
|
||||
stages: std::sync::atomic::AtomicUsize::new(0),
|
||||
releases,
|
||||
});
|
||||
let router = Router::new()
|
||||
.route(
|
||||
SUBJECTS,
|
||||
axum::routing::post(|| async { StatusCode::NO_CONTENT }),
|
||||
)
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
fixture.api.clone(),
|
||||
authorize_workspace_api_request,
|
||||
));
|
||||
let response = observed(crate::db_executor::with_operation_lifetime(
|
||||
source,
|
||||
router.oneshot(Fixture::subject_request("not-executed")),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(release);
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
let body = response_json(response, StatusCode::SERVICE_UNAVAILABLE).await;
|
||||
assert!(
|
||||
!body.to_string().contains("Workspace is deleting"),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn runtime_progress_while_locked(lock: &WriteLock) {
|
||||
let control = tokio::spawn(async { "control progressed" });
|
||||
assert_eq!(observed(control).await.unwrap(), "control progressed");
|
||||
// The timer is the contract, not a guessed wait for request completion.
|
||||
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
|
||||
assert!(
|
||||
!lock.released.load(Ordering::SeqCst),
|
||||
"SQLite watchdog released the lock before runtime progress"
|
||||
);
|
||||
}
|
||||
async fn response_json(response: Response, expected: StatusCode) -> serde_json::Value {
|
||||
let status = response.status();
|
||||
let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes));
|
||||
serde_json::from_slice(&bytes).expect("typed JSON response")
|
||||
}
|
||||
fn subject_count(path: &Path, role: &str) -> i64 {
|
||||
Connection::open(path)
|
||||
.unwrap()
|
||||
.query_row(
|
||||
"SELECT count(*) FROM subjects WHERE role = ?1",
|
||||
[role],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn ticket_create_waiting_for_sqlite_does_not_block_timer_or_control() {
|
||||
let fixture = Fixture::new().await;
|
||||
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
|
||||
let (lifetime, accepted) = admission_observation();
|
||||
let request = Fixture::request(
|
||||
"POST",
|
||||
"/api/w/db-execution/tickets",
|
||||
serde_json::to_value(ticket::NewTicket::new("locked Ticket")).unwrap(),
|
||||
);
|
||||
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
lifetime,
|
||||
fixture.inner_router().oneshot(request),
|
||||
));
|
||||
assert!(futures::poll!(&mut response).is_pending());
|
||||
observed(accepted).await.unwrap();
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert!(
|
||||
futures::poll!(&mut response).is_pending(),
|
||||
"mutation responded before the SQLite lock was released"
|
||||
);
|
||||
let reader = Connection::open(&fixture.api.config.database_path).unwrap();
|
||||
let count: i64 = reader
|
||||
.query_row(
|
||||
"SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1",
|
||||
[WORKSPACE],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(count, 0, "Ticket must not be visible before commit");
|
||||
lock.release();
|
||||
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
|
||||
assert!(body["id"].is_string(), "{body}");
|
||||
// Ticket dispatch can call Runtime callbacks, so its operation owner is
|
||||
// the separate bounded host executor, not the store executor.
|
||||
observed(fixture.api.host_executors.host().unwrap().drain())
|
||||
.await
|
||||
.unwrap();
|
||||
fixture.executor.drain().await.unwrap();
|
||||
let count: i64 = reader.query_row("SELECT count(*) FROM typed_tickets WHERE workspace_id = ?1 AND title = 'locked Ticket'", [WORKSPACE], |row| row.get(0)).unwrap();
|
||||
assert_eq!(count, 1, "one accepted HTTP mutation must commit once");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cold_subjektiv_open_waiting_for_scope_transaction_keeps_runtime_responsive() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
drop(database); // FeatureStorage retains Weak handles: the route must cold-open.
|
||||
let mut lock = WriteLock::acquire(path);
|
||||
let (lifetime, accepted) = admission_observation();
|
||||
let request = Fixture::request("GET", SUBJECTS, serde_json::Value::Null);
|
||||
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
lifetime,
|
||||
fixture.inner_router().oneshot(request),
|
||||
));
|
||||
assert!(futures::poll!(&mut response).is_pending());
|
||||
observed(accepted).await.unwrap();
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert!(
|
||||
futures::poll!(&mut response).is_pending(),
|
||||
"cold open must wait for its SQLite scope transaction"
|
||||
);
|
||||
lock.release();
|
||||
let body = response_json(observed(response).await.unwrap(), StatusCode::OK).await;
|
||||
assert_eq!(body["items"], serde_json::json!([]), "{body}");
|
||||
fixture.executor.drain().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn authenticated_subjektiv_mutation_waits_for_commit_before_response() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
let mut lock = WriteLock::acquire(path.clone());
|
||||
let (mut busy, waiting) = BusyObservation::install(database);
|
||||
let response = fixture.dispatch(Fixture::subject_request("locked-subject"));
|
||||
observed(waiting).await.unwrap();
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert!(
|
||||
!response.is_finished(),
|
||||
"HTTP response must not publish an uncommitted subject"
|
||||
);
|
||||
assert_eq!(subject_count(&path, "locked-subject"), 0);
|
||||
lock.release();
|
||||
busy.resume();
|
||||
let body = response_json(observed(response).await.unwrap(), StatusCode::CREATED).await;
|
||||
assert_eq!(body["role"], "locked-subject", "{body}");
|
||||
assert!(
|
||||
body["memory_fingerprint"].is_string(),
|
||||
"response assembly must finish in the DB operation: {body}"
|
||||
);
|
||||
fixture.executor.drain().await.unwrap();
|
||||
assert_eq!(subject_count(&path, "locked-subject"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_http_waiter_keeps_deletion_pending_until_sql_finishes() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
let mut lock = WriteLock::acquire(path.clone());
|
||||
let (mut busy, waiting) = BusyObservation::install(database);
|
||||
let response = fixture.dispatch(Fixture::subject_request("cancelled-subject"));
|
||||
observed(waiting).await.unwrap(); // Actual SQLite wait, not inferred task scheduling.
|
||||
response.abort();
|
||||
assert!(observed(response).await.unwrap_err().is_cancelled());
|
||||
let admission = fixture.server.admission_for_workspace(WORKSPACE).await;
|
||||
let mut deletion = Box::pin(admission.lock_deletion());
|
||||
assert!(
|
||||
futures::poll!(&mut deletion).is_pending(),
|
||||
"aborting HTTP must not drop the accepted SQL RequestLease"
|
||||
);
|
||||
assert!(
|
||||
admission.admit(false).is_none(),
|
||||
"deletion must fence fresh requests"
|
||||
);
|
||||
assert_eq!(subject_count(&path, "cancelled-subject"), 0);
|
||||
lock.release();
|
||||
busy.resume();
|
||||
fixture.executor.drain().await.unwrap();
|
||||
let deletion = observed(deletion).await;
|
||||
assert_eq!(
|
||||
subject_count(&path, "cancelled-subject"),
|
||||
1,
|
||||
"cancelled waiter must neither cancel nor retry accepted SQL"
|
||||
);
|
||||
drop(deletion);
|
||||
assert!(
|
||||
admission.admit(false).is_some(),
|
||||
"deletion lease cleanup must reopen admission"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_http_mutation_keeps_capacity_until_commit_and_drain() {
|
||||
let fixture = Fixture::new().await;
|
||||
let (database, path) = fixture.feature_database();
|
||||
let mut lock = WriteLock::acquire(path.clone());
|
||||
let (mut busy, waiting) = BusyObservation::install(database);
|
||||
let running = fixture.dispatch(Fixture::subject_request("running-subject"));
|
||||
observed(waiting).await.unwrap();
|
||||
let (lifetime, accepted) = admission_observation();
|
||||
let mut queued = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
lifetime,
|
||||
fixture
|
||||
.inner_router()
|
||||
.oneshot(Fixture::subject_request("queued-subject")),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
observed(accepted).await.unwrap();
|
||||
drop(queued); // Accepted job still owns the only waiting slot.
|
||||
let rejected = observed(
|
||||
fixture
|
||||
.inner_router()
|
||||
.oneshot(Fixture::subject_request("rejected-subject")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await;
|
||||
let error: server_api::RepositoryApiError = serde_json::from_value(body.clone()).unwrap();
|
||||
assert!(
|
||||
error.message.contains("capacity"),
|
||||
"typed overload error: {body}"
|
||||
);
|
||||
assert_eq!(subject_count(&path, "queued-subject"), 0);
|
||||
assert_eq!(subject_count(&path, "rejected-subject"), 0);
|
||||
let mut drained = Box::pin(fixture.executor.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
lock.release();
|
||||
busy.resume();
|
||||
response_json(observed(running).await.unwrap(), StatusCode::CREATED).await;
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(
|
||||
subject_count(&path, "queued-subject"),
|
||||
1,
|
||||
"cancelled accepted mutation persists exactly once"
|
||||
);
|
||||
assert_eq!(
|
||||
subject_count(&path, "rejected-subject"),
|
||||
0,
|
||||
"overload must not execute rejected SQL"
|
||||
);
|
||||
let retry = fixture.dispatch(Fixture::subject_request("after-drain"));
|
||||
response_json(observed(retry).await.unwrap(), StatusCode::CREATED).await;
|
||||
fixture.executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
subject_count(&path, "after-drain"),
|
||||
1,
|
||||
"drain must release actual-job capacity"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn saturated_cancelled_host_jobs_keep_their_budget_but_not_other_fixture_or_cleanup_admission()
|
||||
{
|
||||
use std::sync::{
|
||||
Condvar,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
// A drop guard releases workers even when an assertion panics.
|
||||
struct Release(Arc<(Mutex<bool>, Condvar)>);
|
||||
impl Drop for Release {
|
||||
fn drop(&mut self) {
|
||||
*self.0.0.lock().unwrap() = true;
|
||||
self.0.1.notify_all();
|
||||
}
|
||||
}
|
||||
let owner = HostExecutors::default();
|
||||
let other = HostExecutors::default();
|
||||
let host = owner.host().unwrap();
|
||||
let gate = Arc::new((Mutex::new(false), Condvar::new()));
|
||||
let release = Release(gate.clone());
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let mut waiters = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let host = host.clone();
|
||||
let gate = gate.clone();
|
||||
let completed = completed.clone();
|
||||
let (started, accepted) = tokio::sync::oneshot::channel();
|
||||
waiters.push(tokio::spawn(async move {
|
||||
host.run(move || {
|
||||
started.send(()).unwrap();
|
||||
let mut released = gate.0.lock().unwrap();
|
||||
while !*released {
|
||||
released = gate.1.wait(released).unwrap();
|
||||
}
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
Ok::<_, Error>(())
|
||||
})
|
||||
.await
|
||||
}));
|
||||
accepted.await.unwrap();
|
||||
}
|
||||
for _ in 0..16 {
|
||||
let completed = completed.clone();
|
||||
host.dispatch(move || {
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
for waiter in waiters {
|
||||
waiter.abort();
|
||||
assert!(waiter.await.unwrap_err().is_cancelled());
|
||||
}
|
||||
assert!(matches!(
|
||||
host.run(|| Ok::<_, Error>(())).await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
// Paused Tokio time can advance without completing or releasing actual Host
|
||||
// work. Neither another fixture nor cleanup needs this saturated Host lane.
|
||||
tokio::time::advance(std::time::Duration::from_secs(31)).await;
|
||||
assert_eq!(completed.load(Ordering::SeqCst), 0);
|
||||
other
|
||||
.host()
|
||||
.unwrap()
|
||||
.run(|| Ok::<_, Error>(()))
|
||||
.await
|
||||
.unwrap();
|
||||
owner
|
||||
.cleanup()
|
||||
.unwrap()
|
||||
.run(|| Ok::<_, Error>(()))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
host.run(|| Ok::<_, Error>(())).await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
drop(release);
|
||||
host.drain().await.unwrap();
|
||||
assert_eq!(completed.load(Ordering::SeqCst), 18);
|
||||
host.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
host.drain().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn restore_intent_pins_once_after_write_ownership_without_a_deferred_read_upgrade() {
|
||||
let fixture = Fixture::new().await;
|
||||
let store = fixture.api.config_store.clone();
|
||||
let (mut busy, waiting) = BusyObservation::install_store(store.clone());
|
||||
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
|
||||
let worker = RuntimeWorkerRef::new("restore-runtime", "restore-worker");
|
||||
let pin_store = store.clone();
|
||||
let pin_worker = worker.clone();
|
||||
let mut pinned = Box::pin(fixture.executor.run(move || {
|
||||
pin_store.pin_internal_worker_restore_intent(
|
||||
WORKSPACE,
|
||||
&pin_worker,
|
||||
"restore-domain",
|
||||
Some("observed-before-dispatch"),
|
||||
Some("restore-request"),
|
||||
)
|
||||
}));
|
||||
tokio::select! {
|
||||
result = &mut pinned => panic!("restore intent must wait for write ownership before reading, not fail a deferred read upgrade: {result:?}"),
|
||||
arrived = observed(waiting) => arrived.unwrap(),
|
||||
}
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert!(futures::poll!(&mut pinned).is_pending());
|
||||
lock.release();
|
||||
busy.resume();
|
||||
let request = observed(pinned).await.unwrap();
|
||||
assert_eq!(request.request_id, "restore-request");
|
||||
assert_eq!(
|
||||
request.expected_observation_token,
|
||||
"observed-before-dispatch"
|
||||
);
|
||||
fixture.executor.drain().await.unwrap();
|
||||
let saved = store
|
||||
.get_internal_worker_restore_intent(WORKSPACE, &worker, &request.request_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(saved.request_id, request.request_id);
|
||||
assert_eq!(
|
||||
saved.expected_observation_token,
|
||||
request.expected_observation_token
|
||||
);
|
||||
let count: i64 = store
|
||||
.with_conn(|conn| {
|
||||
Ok(conn.query_row(
|
||||
"SELECT count(*) FROM worker_restore_intents WHERE workspace_id=?1 AND request_id=?2",
|
||||
rusqlite::params![WORKSPACE, request.request_id], |row| row.get(0)
|
||||
)?)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
count, 1,
|
||||
"one admitted intent must persist exactly once, with no operation retry"
|
||||
);
|
||||
}
|
||||
@@ -51,14 +51,13 @@ async fn blocking<T: Send + 'static>(
|
||||
mutation: bool,
|
||||
action: impl FnOnce() -> workspace_drive::Result<T> + Send + 'static,
|
||||
) -> DriveResult<T> {
|
||||
tokio::task::spawn_blocking(action)
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || Ok::<_, Error>(action()))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
if mutation {
|
||||
DriveApiError::new(DriveApiErrorCode::OutcomeUnknown)
|
||||
} else {
|
||||
failure(503)
|
||||
}
|
||||
.map_err(|error| match error {
|
||||
Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503),
|
||||
_ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown),
|
||||
_ => failure(503),
|
||||
})?
|
||||
.map_err(|error| domain_error(error, mutation))
|
||||
}
|
||||
|
||||
@@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> DriveApiError {
|
||||
})
|
||||
}
|
||||
fn mutation_error(error: Error) -> DriveApiError {
|
||||
if matches!(error, Error::Sqlite(_) | Error::Store(_)) {
|
||||
if matches!(
|
||||
error,
|
||||
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
|
||||
) {
|
||||
fail(DriveApiErrorCode::OutcomeUnknown)
|
||||
} else {
|
||||
pre_error(error)
|
||||
@@ -97,23 +100,27 @@ pub(super) async fn authorized_drive(
|
||||
};
|
||||
// Reject missing/read-only grants before upload body reception. This check
|
||||
// is admission only: every later operation rechecks under its own lock.
|
||||
tokio::task::spawn_blocking(move || {
|
||||
drive.check_authorized(write)?;
|
||||
Ok::<_, workspace_drive::Error>(drive)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
.map_err(|error| {
|
||||
fail(match error {
|
||||
workspace_drive::Error::Denied => DriveApiErrorCode::Denied,
|
||||
workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => {
|
||||
DriveApiErrorCode::Conflict
|
||||
}
|
||||
workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound,
|
||||
workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid,
|
||||
_ => DriveApiErrorCode::StorageUnavailable,
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
drive.check_authorized(write)?;
|
||||
Ok::<_, workspace_drive::Error>(drive)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
.map_err(|error| {
|
||||
fail(match error {
|
||||
workspace_drive::Error::Denied => DriveApiErrorCode::Denied,
|
||||
workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => {
|
||||
DriveApiErrorCode::Conflict
|
||||
}
|
||||
workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound,
|
||||
workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid,
|
||||
_ => DriveApiErrorCode::StorageUnavailable,
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn create_grant(
|
||||
@@ -152,9 +159,10 @@ pub(super) async fn create_grant(
|
||||
revoked_at: None,
|
||||
};
|
||||
let store = api.store.clone();
|
||||
tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant))
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || store.create_workspace_drive_grant(&grant))
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
|
||||
.map_err(mutation_error)
|
||||
}
|
||||
|
||||
@@ -175,13 +183,12 @@ pub(super) async fn revoke_grant(
|
||||
actor.account_id.clone(),
|
||||
);
|
||||
let store = api.store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
store.revoke_workspace_drive_grant(&workspace, &grant, &actor)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
|
||||
.map_err(mutation_error)?
|
||||
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor))
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
|
||||
}
|
||||
|
||||
fn validate_grant_id(id: &str) -> DriveResult<()> {
|
||||
@@ -212,12 +219,14 @@ pub(super) async fn list_grants(
|
||||
}
|
||||
let workspace = workspace_id.to_owned();
|
||||
let store = api.store.clone();
|
||||
let mut grants = tokio::task::spawn_blocking(move || {
|
||||
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
.map_err(pre_error)?;
|
||||
let mut grants = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let more = grants.len() > limit;
|
||||
grants.truncate(limit);
|
||||
let next_after = if more {
|
||||
|
||||
@@ -724,7 +724,7 @@ async fn catalog_case_dispatch_authenticates_and_routes_the_canonical_workspace_
|
||||
reason: ticket::TicketClosureReason::Completed,
|
||||
},
|
||||
);
|
||||
let catalog = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
|
||||
let catalog = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
|
||||
// Retain only this in-process fixture: do not create a real Runtime or a
|
||||
// background Orchestrator hook while testing catalog authentication/routing.
|
||||
catalog
|
||||
|
||||
@@ -529,7 +529,7 @@ async fn runtime_tag_mutations_reject_invalid_proofs_and_changed_binding_authori
|
||||
*tampered.body_mut() = Body::from(format!("{} ", input));
|
||||
runtime_tag_response(app.clone(), tampered, StatusCode::UNAUTHORIZED).await;
|
||||
}
|
||||
let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap();
|
||||
let original = api
|
||||
.store
|
||||
.get_workspace_runtime_binding(TEST_WORKSPACE_ID, "runtime-tags")
|
||||
|
||||
@@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> Failure {
|
||||
fn mutation_error(error: Error) -> Failure {
|
||||
// Storage/service failures at a mutation boundary may include a commit
|
||||
// acknowledgement failure. Never describe those as a guaranteed rollback.
|
||||
if matches!(&error, Error::Sqlite(_) | Error::Store(_)) {
|
||||
if matches!(
|
||||
&error,
|
||||
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
|
||||
) {
|
||||
unknown()
|
||||
} else {
|
||||
pre_error(error)
|
||||
@@ -186,9 +189,23 @@ pub(super) async fn get(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ConfigResult<Option<WorkspaceConfigAttachment>> {
|
||||
let _guard = lock(api, worker).await?;
|
||||
let grant = grant_for_worker(api, worker)?;
|
||||
Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true)))
|
||||
let guard = lock(api, worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let worker = worker.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
let grant = grant_for_worker(&api, &worker)?;
|
||||
Ok::<_, Failure>(
|
||||
active_link(&api, &worker, &grant, None)?
|
||||
.map(|link| attachment(&grant, &link, true)),
|
||||
)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
}
|
||||
pub(super) async fn attach(
|
||||
api: &WorkspaceApi,
|
||||
@@ -198,46 +215,59 @@ pub(super) async fn attach(
|
||||
if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) {
|
||||
return Err(fail(400, "invalid_alias"));
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let grant = grant_for_worker(api, worker)?;
|
||||
let access = request.access.unwrap_or(grant.access);
|
||||
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if let Some(mut link) = active_link(api, worker, &grant, None)? {
|
||||
// Same active alias/identity is idempotent; explicit write never silently downgrades.
|
||||
if request.access == Some(Access::ReadWrite)
|
||||
&& effective_access(&grant, &link) != Access::ReadWrite
|
||||
{
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly {
|
||||
link.capabilities = access_capabilities(access);
|
||||
link = api
|
||||
.store
|
||||
.attach_worker_workdir(&link)
|
||||
.map_err(mutation_error)?;
|
||||
}
|
||||
return Ok(attachment(&grant, &link, true));
|
||||
}
|
||||
let link = api
|
||||
.store
|
||||
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
|
||||
connection_id: String::new(),
|
||||
workspace_id: grant.workspace_id.clone(),
|
||||
worker: worker.clone(),
|
||||
workdir_id: grant.working_directory_id.clone(),
|
||||
alias: ALIAS.into(),
|
||||
capabilities: access_capabilities(access),
|
||||
linked_at: now_registry_timestamp(),
|
||||
unlinked_at: None,
|
||||
let guard = lock(api, worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let worker = worker.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
let grant = grant_for_worker(&api, &worker)?;
|
||||
let access = request.access.unwrap_or(grant.access);
|
||||
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if let Some(mut link) = active_link(&api, &worker, &grant, None)? {
|
||||
// Same active alias/identity is idempotent; explicit write never silently downgrades.
|
||||
if request.access == Some(Access::ReadWrite)
|
||||
&& effective_access(&grant, &link) != Access::ReadWrite
|
||||
{
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if access == Access::ReadOnly
|
||||
&& effective_access(&grant, &link) != Access::ReadOnly
|
||||
{
|
||||
link.capabilities = access_capabilities(access);
|
||||
link = api
|
||||
.store
|
||||
.attach_worker_workdir(&link)
|
||||
.map_err(mutation_error)?;
|
||||
}
|
||||
return Ok(attachment(&grant, &link, true));
|
||||
}
|
||||
let link = api
|
||||
.store
|
||||
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
|
||||
connection_id: String::new(),
|
||||
workspace_id: grant.workspace_id.clone(),
|
||||
worker: worker.clone(),
|
||||
workdir_id: grant.working_directory_id.clone(),
|
||||
alias: ALIAS.into(),
|
||||
capabilities: access_capabilities(access),
|
||||
linked_at: now_registry_timestamp(),
|
||||
unlinked_at: None,
|
||||
})
|
||||
.map_err(mutation_error)?;
|
||||
// No Runtime filesystem/session synchronization for a logical config target.
|
||||
api.worker_projection
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(attachment(&grant, &link, false))
|
||||
})())
|
||||
})
|
||||
.map_err(mutation_error)?;
|
||||
// No Runtime filesystem/session synchronization for a logical config target.
|
||||
api.worker_projection
|
||||
.refresh(worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(attachment(&grant, &link, false))
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
fn resolve(
|
||||
api: &WorkspaceApi,
|
||||
@@ -313,9 +343,22 @@ pub(super) async fn observe(
|
||||
for path in &request.paths {
|
||||
path_valid(path, true)?;
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, _guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -427,9 +470,22 @@ pub(super) async fn read(
|
||||
request: WorkspaceConfigReadRequest,
|
||||
) -> ConfigResult<WorkspaceConfigReadResponse> {
|
||||
path_valid(&request.path, false)?;
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, _guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -482,12 +538,25 @@ pub(super) async fn commit(
|
||||
server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?,
|
||||
}
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
if effective_access(&grant, &link) != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
if effective_access(&grant, &link) != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -501,17 +570,19 @@ pub(super) async fn commit(
|
||||
// Evaluation/semantic validation are pure preparation. Bound this stage
|
||||
// without ever dispatching persistence in a timed-out background task.
|
||||
let preparation_api = api.clone();
|
||||
let preparation = tokio::task::spawn_blocking(move || {
|
||||
prepare_workspace_config_tree(
|
||||
let executor = api.store.db_executor();
|
||||
let preparation = executor.run(move || {
|
||||
// Timeout/cancellation releases exclusion only after actual preparation ends.
|
||||
let candidate = prepare_workspace_config_tree(
|
||||
&preparation_api,
|
||||
&preparation_api.config.workspace_id,
|
||||
&canonical,
|
||||
)
|
||||
)?;
|
||||
Ok::<_, Error>((candidate, guard))
|
||||
});
|
||||
let candidate = tokio::time::timeout(DEADLINE, preparation)
|
||||
let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation)
|
||||
.await
|
||||
.map_err(|_| fail(503, "evaluation_timeout"))?
|
||||
.map_err(|_| fail(500, "evaluation_unavailable"))?
|
||||
.map_err(pre_error)?;
|
||||
if candidate.base_digest != state.snapshot.digest
|
||||
|| candidate.base_toolchain_fingerprint != state.contract.fingerprint
|
||||
@@ -520,15 +591,78 @@ pub(super) async fn commit(
|
||||
{
|
||||
return Err(fail(409, "stale_validator"));
|
||||
}
|
||||
// Recheck durable authority before the first auxiliary/canonical effects.
|
||||
let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?;
|
||||
if current_grant != grant || current_link.capabilities != link.capabilities {
|
||||
return Err(fail(409, "authority_changed"));
|
||||
let persistence_api = api.clone();
|
||||
let worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let expected_grant = grant.clone();
|
||||
let expected_link = link.clone();
|
||||
let (state, publication) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (current_grant, current_link) =
|
||||
resolve(&persistence_api, &worker, &connection_id)?;
|
||||
if current_grant != expected_grant
|
||||
|| current_link.capabilities != expected_link.capabilities
|
||||
{
|
||||
return Err(fail(409, "authority_changed"));
|
||||
}
|
||||
// Preserve the shared helper's ordering and atomic CAS, but never
|
||||
// run its remote Runtime publication on the sole DB worker.
|
||||
project_repository_access_candidate(
|
||||
&*persistence_api.store,
|
||||
&persistence_api.repository_secrets,
|
||||
&persistence_api.config.workspace_id,
|
||||
&candidate,
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
let state = persistence_api
|
||||
.config_store
|
||||
.commit_evaluated_workspace_config(
|
||||
&persistence_api.config.workspace_id,
|
||||
&candidate,
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
let publication = persistence_api
|
||||
.prompt_projection_cache
|
||||
.resolve(&persistence_api.config.workspace_id, &state)
|
||||
.ok()
|
||||
.and_then(|projection| {
|
||||
// Dispatch from the actual persistence owner even if its
|
||||
// waiter was cancelled. Runtime I/O owns exclusion, not
|
||||
// a DB worker or an open SQL transaction.
|
||||
let (complete, publication) = tokio::sync::oneshot::channel();
|
||||
let runtime = persistence_api.runtime.clone();
|
||||
let admission = persistence_api.host_executors.host().and_then(|executor| {
|
||||
executor.dispatch(move || {
|
||||
let _guard = guard;
|
||||
let _diagnostics = runtime
|
||||
.observe_workspace_prompt_projection((*projection).clone());
|
||||
let _ = complete.send(());
|
||||
})
|
||||
});
|
||||
match admission {
|
||||
Ok(()) => Some(publication),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
workspace_id = %persistence_api.config.workspace_id,
|
||||
%error,
|
||||
"Workspace config committed but Runtime publication was not admitted"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok((state, publication))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)??;
|
||||
if let Some(publication) = publication {
|
||||
// Publication is best effort and cannot turn an acknowledged CAS into an error.
|
||||
let _ = publication.await;
|
||||
}
|
||||
// From this boundary repository credential projection may have side effects.
|
||||
// Never falsely claim rollback or automatically replay a failed/uncertain save.
|
||||
let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(WorkspaceConfigCommitResponse {
|
||||
validator: validator(&grant, &link, &state, &schema_fingerprint, ""),
|
||||
digest: state.snapshot.digest,
|
||||
@@ -544,37 +678,61 @@ pub(super) async fn create_grant(
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id);
|
||||
let _guard = lock(api, &worker).await?;
|
||||
let summary = api
|
||||
.runtime
|
||||
.worker(&worker)
|
||||
.map_err(|_| fail(404, "worker_not_found"))?;
|
||||
let guard = lock(api, &worker).await?;
|
||||
let runtime = api.runtime.clone();
|
||||
let lookup_worker = worker.clone();
|
||||
let (summary, guard) = api
|
||||
.host_executors
|
||||
.host()
|
||||
.map_err(pre_error)?
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let summary = runtime
|
||||
.worker(&lookup_worker)
|
||||
.map_err(|_| fail(404, "worker_not_found"))?;
|
||||
Ok::<_, Failure>((summary, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
if summary.workspace.workspace_id.as_deref() != Some(workspace_id) {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if let Some(grant) = api
|
||||
.store
|
||||
.current_workspace_config_grant(workspace_id, &worker)
|
||||
.map_err(pre_error)?
|
||||
{
|
||||
if grant.access != request.access {
|
||||
return Err(fail(409, "grant_exists_revoke_first"));
|
||||
}
|
||||
return Ok(grant);
|
||||
}
|
||||
let grant = WorkspaceConfigGrantResponse {
|
||||
grant_id: Uuid::now_v7().to_string(),
|
||||
workspace_id: workspace_id.into(),
|
||||
runtime_id: worker.runtime_id,
|
||||
worker_id: worker.worker_id,
|
||||
working_directory_id: Uuid::now_v7().to_string(),
|
||||
access: request.access,
|
||||
revoked: false,
|
||||
};
|
||||
api.store
|
||||
.create_workspace_config_grant(&grant, &actor.account_id)
|
||||
.map_err(mutation_error)?;
|
||||
Ok(grant)
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let account_id = actor.account_id.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
if let Some(grant) = api
|
||||
.store
|
||||
.current_workspace_config_grant(&workspace_id, &worker)
|
||||
.map_err(pre_error)?
|
||||
{
|
||||
if grant.access != request.access {
|
||||
return Err(fail(409, "grant_exists_revoke_first"));
|
||||
}
|
||||
return Ok(grant);
|
||||
}
|
||||
let grant = WorkspaceConfigGrantResponse {
|
||||
grant_id: Uuid::now_v7().to_string(),
|
||||
workspace_id,
|
||||
runtime_id: worker.runtime_id,
|
||||
worker_id: worker.worker_id,
|
||||
working_directory_id: Uuid::now_v7().to_string(),
|
||||
access: request.access,
|
||||
revoked: false,
|
||||
};
|
||||
api.store
|
||||
.create_workspace_config_grant(&grant, &account_id)
|
||||
.map_err(mutation_error)?;
|
||||
Ok(grant)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
pub(super) async fn revoke_grant(
|
||||
api: &WorkspaceApi,
|
||||
@@ -585,39 +743,60 @@ pub(super) async fn revoke_grant(
|
||||
require_workspace_owner(api, workspace_id, actor, "Workspace config grants")
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let lookup_api = api.clone();
|
||||
let lookup_workspace_id = workspace_id.to_owned();
|
||||
let lookup_grant_id = grant_id.to_owned();
|
||||
let mut grant = api
|
||||
.store
|
||||
.get_workspace_config_grant(workspace_id, grant_id)
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
lookup_api
|
||||
.store
|
||||
.get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id)
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
.ok_or_else(|| fail(404, "not_found"))?;
|
||||
let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id);
|
||||
let _guard = lock(api, &worker).await?;
|
||||
api.store
|
||||
.revoke_workspace_config_grant(workspace_id, grant_id)
|
||||
.map_err(mutation_error)?;
|
||||
grant.revoked = true;
|
||||
// Revocation is durable before unlink; a failed cleanup cannot restore access.
|
||||
if let Some(link) = api
|
||||
.store
|
||||
.list_workdir_worker_links(workspace_id, &grant.working_directory_id)
|
||||
.map_err(|_| unknown())?
|
||||
.into_iter()
|
||||
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
|
||||
{
|
||||
api.store
|
||||
.detach_worker_workdir_connection(
|
||||
workspace_id,
|
||||
&worker,
|
||||
&link.alias,
|
||||
&link.connection_id,
|
||||
&now_registry_timestamp(),
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
}
|
||||
api.worker_projection
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(grant)
|
||||
let guard = lock(api, &worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let grant_id = grant_id.to_owned();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
api.store
|
||||
.revoke_workspace_config_grant(&workspace_id, &grant_id)
|
||||
.map_err(mutation_error)?;
|
||||
grant.revoked = true;
|
||||
// Revocation is durable before unlink; a failed cleanup cannot restore access.
|
||||
if let Some(link) = api
|
||||
.store
|
||||
.list_workdir_worker_links(&workspace_id, &grant.working_directory_id)
|
||||
.map_err(|_| unknown())?
|
||||
.into_iter()
|
||||
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
|
||||
{
|
||||
api.store
|
||||
.detach_worker_workdir_connection(
|
||||
&workspace_id,
|
||||
&worker,
|
||||
&link.alias,
|
||||
&link.connection_id,
|
||||
&now_registry_timestamp(),
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
}
|
||||
api.worker_projection
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(grant)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
|
||||
pub(super) fn summary(
|
||||
@@ -655,7 +834,11 @@ pub(super) fn summary(
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn workspace_config_mutation_storage_failures_are_conservatively_unknown() {
|
||||
fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() {
|
||||
assert_eq!(
|
||||
mutation_error(Error::DbOperationOutcomeUnknown).classification,
|
||||
Classification::Unknown
|
||||
);
|
||||
assert_eq!(
|
||||
mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification,
|
||||
Classification::Unknown
|
||||
|
||||
@@ -318,7 +318,7 @@ mod workspace_config_integration {
|
||||
};
|
||||
for scoped in [false, true] {
|
||||
let app = if scoped {
|
||||
workspace_server_router(WorkspaceServerApi::new(
|
||||
workspace_server_router(WorkspaceServerApi::new_for_test(
|
||||
api.config.clone(),
|
||||
api.store.clone(),
|
||||
))
|
||||
@@ -803,9 +803,9 @@ mod workspace_config_integration {
|
||||
.supports(workdir::WorkdirSessionCapability::Command)
|
||||
);
|
||||
let lock = current_worker_session_lock(api, &fixture.worker);
|
||||
let _guard = lock.lock().await;
|
||||
let _guard = Arc::new(lock.lock_owned().await);
|
||||
assert!(
|
||||
open_current_worker_workdir_session_locked(api, &fixture.worker, &link)
|
||||
open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
@@ -1385,7 +1385,7 @@ mod workspace_config_integration {
|
||||
let api = &fixture.api;
|
||||
let granted = grant(api, &fixture.worker, Access::ReadWrite).await;
|
||||
let attached = attach(api, &fixture.worker).await;
|
||||
let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap();
|
||||
let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap();
|
||||
assert_eq!(
|
||||
restarted
|
||||
.get_workspace_config_grant(TEST_WORKSPACE_ID, &granted.grant_id)
|
||||
|
||||
@@ -800,7 +800,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
||||
let base = format!("http://{addr}");
|
||||
let mut config = test_server_config(root.path());
|
||||
config.backend_base_url = Some("server-test".into());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
let captured = Arc::new(Mutex::new(None));
|
||||
let abort = Arc::new(Mutex::new(None));
|
||||
let factory = ConfigCaptureFactory {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -263,7 +263,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64
|
||||
let tags_before = retained.as_ref().map(|_| classification_rows(&conn));
|
||||
let auth_before = fixture_auth_rows(&conn);
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?,93);
|
||||
assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11);
|
||||
@@ -490,7 +490,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history
|
||||
let metadata_before = stored_fixture_rows(&conn, "typed_ticket_case_metadata");
|
||||
let auth_before = fixture_auth_rows(&conn);
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?,93);
|
||||
|
||||
@@ -750,7 +750,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn retry_keeps_resolved_config_and_credential_candidates_after_fallback_moves() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace".to_string(),
|
||||
owner_account_id: "owner-account".to_string(),
|
||||
|
||||
@@ -925,7 +925,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn seeded_store() -> (SqliteWorkspaceStore, WorkdirRegistryRecord) {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-a".to_string(),
|
||||
|
||||
@@ -5,7 +5,7 @@ use chrono::Utc;
|
||||
use protocol::subscription::{
|
||||
EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot,
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
use tokio::sync::{Semaphore, broadcast};
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use crate::runtime_subscription::{
|
||||
@@ -59,7 +59,14 @@ pub struct WorkerProjectionService {
|
||||
events: broadcast::Sender<WorkerProjectionEvent>,
|
||||
ordering: Mutex<()>,
|
||||
attach_gate: tokio::sync::Mutex<()>,
|
||||
tasks: Mutex<HashMap<String, JoinHandle<()>>>,
|
||||
tasks: Mutex<HashMap<String, RuntimeProjectionLifetime>>,
|
||||
}
|
||||
|
||||
struct RuntimeProjectionLifetime {
|
||||
consumer: Option<JoinHandle<()>>,
|
||||
// The permit belongs to the actual DB closure, not its cancellable async waiter.
|
||||
// Keep this owner in the map even if attachment initialization is cancelled.
|
||||
operations: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl WorkerProjectionService {
|
||||
@@ -75,17 +82,89 @@ impl WorkerProjectionService {
|
||||
})
|
||||
}
|
||||
|
||||
/// Stops consumers but does not claim that accepted database work has finished.
|
||||
/// Async shutdown callers must use `shutdown_async` to drain accepted operations.
|
||||
pub fn shutdown(&self) {
|
||||
for (_, task) in self
|
||||
for lifetime in self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.drain()
|
||||
.values()
|
||||
{
|
||||
task.abort();
|
||||
if let Some(task) = &lifetime.consumer {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn shutdown_async(&self) {
|
||||
let _attach = self.attach_gate.lock().await;
|
||||
self.shutdown();
|
||||
let runtime_ids = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for runtime_id in runtime_ids {
|
||||
self.drain_runtime(&runtime_id).await;
|
||||
}
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.clear();
|
||||
}
|
||||
|
||||
async fn drain_runtime(&self, runtime_id: &str) {
|
||||
let previous = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.get_mut(runtime_id)
|
||||
.map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone()));
|
||||
if let Some((consumer, operations)) = previous {
|
||||
if let Some(consumer) = consumer {
|
||||
consumer.abort();
|
||||
let _ = consumer.await;
|
||||
}
|
||||
// Aborting the waiter cannot abort a queued/running synchronous job.
|
||||
// Cancellation here leaves the gate in the map for the next drain.
|
||||
let _drained = operations
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("operation gate closed");
|
||||
}
|
||||
}
|
||||
|
||||
// Caller holds attach_gate. Install ownership before any initialization await.
|
||||
async fn replace_runtime(&self, runtime_id: &str) -> Arc<Semaphore> {
|
||||
self.drain_runtime(runtime_id).await;
|
||||
let operations = Arc::new(Semaphore::new(1));
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.insert(
|
||||
runtime_id.to_string(),
|
||||
RuntimeProjectionLifetime {
|
||||
consumer: None,
|
||||
operations: operations.clone(),
|
||||
},
|
||||
);
|
||||
operations
|
||||
}
|
||||
|
||||
pub async fn subscribe_ordered_async(
|
||||
self: &Arc<Self>,
|
||||
limit: usize,
|
||||
) -> crate::Result<WorkerProjectionSubscription> {
|
||||
let service = self.clone();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.subscribe_ordered(limit))
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn subscribe_ordered(&self, limit: usize) -> crate::Result<WorkerProjectionSubscription> {
|
||||
let _ordering = self
|
||||
.ordering
|
||||
@@ -125,19 +204,10 @@ impl WorkerProjectionService {
|
||||
broker: &RuntimeSubscriptionBroker,
|
||||
runtime_id: &str,
|
||||
) -> Result<(), RuntimeSubscriptionBrokerError> {
|
||||
// Serialize replacement and wait for the old consumer to stop before the new
|
||||
// lifetime can apply its snapshot. Late notifications can therefore never
|
||||
// cross a Runtime ownership boundary.
|
||||
// Serialize replacement, stop admission by aborting the consumer, and then
|
||||
// drain ownership held by the actual SQL jobs before the new snapshot.
|
||||
let _attach = self.attach_gate.lock().await;
|
||||
let previous = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.remove(runtime_id);
|
||||
if let Some(previous) = previous {
|
||||
previous.abort();
|
||||
let _ = previous.await;
|
||||
}
|
||||
let operations = self.replace_runtime(runtime_id).await;
|
||||
|
||||
let mut subscription =
|
||||
broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?;
|
||||
@@ -148,12 +218,25 @@ impl WorkerProjectionService {
|
||||
break;
|
||||
};
|
||||
let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. });
|
||||
if let Err(error) = self.apply_broker_event(runtime_id, event) {
|
||||
if let Err(error) = self
|
||||
.apply_broker_event_owned_async(runtime_id, event, operations.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
runtime_id,
|
||||
error = %error,
|
||||
"failed to initialize durable Worker projection; restarting from a fresh snapshot"
|
||||
);
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbCapacityExceeded
|
||||
) {
|
||||
// Do not automatically replay a possibly committed mutation or
|
||||
// spin on executor overload; attachment must be retried explicitly.
|
||||
return Err(RuntimeSubscriptionBrokerError::Closed);
|
||||
}
|
||||
subscription.restart()?;
|
||||
continue;
|
||||
}
|
||||
@@ -169,12 +252,27 @@ impl WorkerProjectionService {
|
||||
let Some(service) = service.upgrade() else {
|
||||
return;
|
||||
};
|
||||
if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) {
|
||||
if let Err(error) = service
|
||||
.apply_broker_event_owned_async(
|
||||
task_runtime_id.as_str(),
|
||||
event,
|
||||
operations.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
runtime_id = %task_runtime_id,
|
||||
error = %error,
|
||||
"failed to update durable Worker projection; restarting from a fresh snapshot"
|
||||
);
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbCapacityExceeded
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if subscription.restart().is_err() {
|
||||
return;
|
||||
}
|
||||
@@ -184,10 +282,61 @@ impl WorkerProjectionService {
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.insert(runtime_id, task);
|
||||
.get_mut(&runtime_id)
|
||||
.expect("Runtime lifetime installed before initialization")
|
||||
.consumer = Some(task);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn seed_observation_async(
|
||||
self: &Arc<Self>,
|
||||
runtime_id: &str,
|
||||
worker: &protocol::subscription::SubscriptionWorker,
|
||||
observed_at: &str,
|
||||
) -> crate::Result<()> {
|
||||
let service = self.clone();
|
||||
let runtime_id = runtime_id.to_string();
|
||||
let worker = worker.clone();
|
||||
let observed_at = observed_at.to_string();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.seed_observation(&runtime_id, &worker, &observed_at))
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn refresh_async(self: &Arc<Self>, worker: &RuntimeWorkerRef) -> crate::Result<()> {
|
||||
let service = self.clone();
|
||||
let worker = worker.clone();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.refresh(&worker))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn apply_broker_event_owned_async(
|
||||
self: &Arc<Self>,
|
||||
runtime_id: &str,
|
||||
event: BrokerSubscriptionEvent,
|
||||
operations: Arc<Semaphore>,
|
||||
) -> crate::Result<()> {
|
||||
let operation = operations
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("operation gate closed");
|
||||
let service = self.clone();
|
||||
let runtime_id = runtime_id.to_string();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
// The permit must outlive SQL commit AND ordered publication, even
|
||||
// when the executor result receiver has been dropped by cancellation.
|
||||
let _operation = operation;
|
||||
service.apply_broker_event(&runtime_id, event)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn seed_observation(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -313,7 +462,11 @@ mod tests {
|
||||
tempfile::TempDir,
|
||||
) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap());
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db"))
|
||||
.unwrap()
|
||||
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".to_string(),
|
||||
@@ -355,6 +508,266 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn observation(
|
||||
state: protocol::subscription::SubscriptionWorkerState,
|
||||
) -> protocol::subscription::SubscriptionWorker {
|
||||
protocol::subscription::SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(),
|
||||
runtime_id: Some("runtime-a".to_string()),
|
||||
resource_key: None,
|
||||
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
|
||||
worker_state: None,
|
||||
state,
|
||||
has_running_internal_workers: false,
|
||||
workspace_id: Some(WORKSPACE_ID.to_string()),
|
||||
display_name: None,
|
||||
profile: None,
|
||||
job: None,
|
||||
workdir_attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// Real second-connection contention; the controller releases automatically on
|
||||
// failure, so a regression that blocks the Tokio thread cannot hang the suite.
|
||||
struct WriteLock {
|
||||
release: Option<std::sync::mpsc::Sender<()>>,
|
||||
controller: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl WriteLock {
|
||||
fn acquire(path: std::path::PathBuf) -> Self {
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
let (locked, ready) = std::sync::mpsc::channel();
|
||||
let controller = std::thread::spawn(move || {
|
||||
let connection = rusqlite::Connection::open(path).unwrap();
|
||||
connection.execute_batch("BEGIN IMMEDIATE").unwrap();
|
||||
locked.send(()).unwrap();
|
||||
let _ = released.recv_timeout(std::time::Duration::from_secs(3));
|
||||
connection.execute_batch("ROLLBACK").unwrap();
|
||||
});
|
||||
ready.recv().unwrap();
|
||||
Self {
|
||||
release: Some(release),
|
||||
controller: Some(controller),
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&mut self) {
|
||||
if let Some(release) = self.release.take() {
|
||||
let _ = release.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WriteLock {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
if let Some(controller) = self.controller.take() {
|
||||
controller.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_sql(service: &WorkerProjectionService) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
// With the second connection holding the write lock, a held ordering
|
||||
// gate identifies the accepted synchronous mutation, not a fake future.
|
||||
while service.ordering.try_lock().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("projection job did not enter synchronous SQL");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload()
|
||||
{
|
||||
use protocol::subscription::SubscriptionWorkerState;
|
||||
let (store, service, temp) = fixture().await;
|
||||
store
|
||||
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
|
||||
.unwrap();
|
||||
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
|
||||
assert_eq!(subscription.take_snapshot().len(), 1);
|
||||
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
|
||||
let publishing_service = service.clone();
|
||||
let publication = tokio::spawn(async move {
|
||||
publishing_service
|
||||
.seed_observation_async(
|
||||
"runtime-a",
|
||||
&observation(SubscriptionWorkerState::Idle),
|
||||
"2",
|
||||
)
|
||||
.await
|
||||
});
|
||||
wait_for_sql(&service).await;
|
||||
let started = std::time::Instant::now();
|
||||
let (control, controlled) = tokio::sync::oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
control.send("control progressed").unwrap();
|
||||
});
|
||||
let (_, control_result) = tokio::join!(
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)),
|
||||
controlled,
|
||||
);
|
||||
assert_eq!(control_result.unwrap(), "control progressed");
|
||||
assert!(
|
||||
started.elapsed() < std::time::Duration::from_secs(1),
|
||||
"SQLite blocked the current-thread executor"
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
lock.release();
|
||||
publication.await.unwrap().unwrap();
|
||||
let committed = subscription.recv().await.unwrap();
|
||||
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
|
||||
let snapshot = snapshot.take_snapshot();
|
||||
assert_eq!(
|
||||
committed.changes,
|
||||
vec![WorkerCatalogChange::Upsert(snapshot[0].clone())]
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot[0].observation.as_ref().unwrap().worker.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
service
|
||||
.seed_observation_async(
|
||||
"runtime-a",
|
||||
&observation(SubscriptionWorkerState::Running),
|
||||
"3",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Delayed readers receive the first transaction's payload, not a re-query
|
||||
// of mutable authority after a later commit.
|
||||
let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else {
|
||||
panic!("expected upsert");
|
||||
};
|
||||
assert_eq!(
|
||||
first.observation.as_ref().unwrap().worker.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
service
|
||||
.refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a"))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() {
|
||||
use crate::runtime_subscription::BrokerSubscriptionEvent;
|
||||
use protocol::subscription::SubscriptionWorkerState;
|
||||
let (store, service, temp) = fixture().await;
|
||||
store
|
||||
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
|
||||
.unwrap();
|
||||
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
|
||||
subscription.take_snapshot();
|
||||
let operations = service.replace_runtime("runtime-a").await;
|
||||
let old_operations = operations.clone();
|
||||
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
|
||||
let old_service = service.clone();
|
||||
let consumer = tokio::spawn(async move {
|
||||
old_service
|
||||
.apply_broker_event_owned_async(
|
||||
"runtime-a",
|
||||
BrokerSubscriptionEvent::Event {
|
||||
payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted {
|
||||
worker: observation(SubscriptionWorkerState::Running),
|
||||
},
|
||||
},
|
||||
operations,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
service
|
||||
.tasks
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("runtime-a")
|
||||
.unwrap()
|
||||
.consumer = Some(consumer);
|
||||
wait_for_sql(&service).await;
|
||||
let replacing_service = service.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
let _attach = replacing_service.attach_gate.lock().await;
|
||||
replacing_service.replace_runtime("runtime-a").await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
replacement.abort();
|
||||
assert!(replacement.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(
|
||||
old_operations.available_permits(),
|
||||
0,
|
||||
"cancelled drain lost accepted SQL ownership"
|
||||
);
|
||||
// A second replacement must still observe the old gate after cancellation.
|
||||
let replacing_service = service.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
let _attach = replacing_service.attach_gate.lock().await;
|
||||
let operations = replacing_service.replace_runtime("runtime-a").await;
|
||||
replacing_service
|
||||
.apply_broker_event_owned_async(
|
||||
"runtime-a",
|
||||
BrokerSubscriptionEvent::Snapshot {
|
||||
snapshot: protocol::subscription::SubscriptionSnapshot::Workers {
|
||||
workers: vec![observation(SubscriptionWorkerState::Idle)],
|
||||
},
|
||||
},
|
||||
operations,
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
lock.release();
|
||||
replacement.await.unwrap().unwrap();
|
||||
assert_eq!(old_operations.available_permits(), 1);
|
||||
let old_commit = subscription.recv().await.unwrap();
|
||||
let new_commit = subscription.recv().await.unwrap();
|
||||
for (commit, expected) in [
|
||||
(old_commit, SubscriptionWorkerState::Running),
|
||||
(new_commit, SubscriptionWorkerState::Idle),
|
||||
] {
|
||||
let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else {
|
||||
panic!("expected one committed upsert");
|
||||
};
|
||||
assert_eq!(record.observation.as_ref().unwrap().worker.state, expected);
|
||||
}
|
||||
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
|
||||
assert_eq!(
|
||||
snapshot.take_snapshot()[0]
|
||||
.observation
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.worker
|
||||
.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
service.shutdown_async().await;
|
||||
assert!(service.tasks.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn snapshot_registration_is_atomic_with_concurrent_publication() {
|
||||
let (store, service, _temp) = fixture().await;
|
||||
|
||||
@@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError {
|
||||
Replay,
|
||||
#[error("source Worker is not a current member of this Workspace Runtime catalog")]
|
||||
WorkerCatalogMembership,
|
||||
#[error("source proof authority is unavailable")]
|
||||
Unavailable,
|
||||
#[error("source proof authority failed: {0}")]
|
||||
Authority(String),
|
||||
}
|
||||
|
||||
impl From<crate::Error> for WorkerMutationSourceProofError {
|
||||
fn from(error: crate::Error) -> Self {
|
||||
match error {
|
||||
crate::Error::DbCapacityExceeded
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::Sqlite(_) => Self::Unavailable,
|
||||
other => Self::Authority(other.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn verify_worker_remove_source(
|
||||
api: &WorkspaceApi,
|
||||
proof: PresentedWorkerMutationSourceProof<'_>,
|
||||
@@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with(
|
||||
let trusted = store
|
||||
.get_workspace_runtime_binding(&config.workspace_id, &unverified.iss)
|
||||
.await
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?
|
||||
.map_err(WorkerMutationSourceProofError::from)?
|
||||
.filter(|record| record.revoked_at.is_none())
|
||||
.ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?;
|
||||
let public_key = trusted.public_key.as_str();
|
||||
@@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with(
|
||||
runtime_id: claims.iss.clone(),
|
||||
worker_id: claims.worker_id.clone(),
|
||||
};
|
||||
let member = store
|
||||
.get_worker_registry(&config.workspace_id, &worker)
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
if member.is_none() {
|
||||
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
|
||||
}
|
||||
let workspace_id = config.workspace_id.clone();
|
||||
let catalog_store = Arc::clone(store);
|
||||
store
|
||||
.require_current_worker_singleton_owner(&config.workspace_id, &worker)
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
let member = catalog_store
|
||||
.get_worker_registry(&workspace_id, &worker)
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
if member.is_none() {
|
||||
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
|
||||
}
|
||||
catalog_store
|
||||
.require_current_worker_singleton_owner(&workspace_id, &worker)
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
|
||||
let consumed_at = chrono::Utc::now().to_rfc3339();
|
||||
let consumed = store
|
||||
@@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with(
|
||||
&consumed_at,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
if !consumed {
|
||||
return Err(WorkerMutationSourceProofError::Replay);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use worker_runtime::auth::{
|
||||
RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims,
|
||||
RuntimeRequestSourceExpectation, decode_runtime_request_source_claims,
|
||||
@@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError {
|
||||
pub(crate) exp: Option<i64>,
|
||||
pub(crate) now: Option<i64>,
|
||||
pub(crate) time_delta_seconds: Option<u64>,
|
||||
/// Operational unavailability is not a proof or principal rejection.
|
||||
pub(crate) availability: bool,
|
||||
}
|
||||
|
||||
impl RuntimeRequestProofError {
|
||||
pub fn is_availability_error(&self) -> bool {
|
||||
self.availability
|
||||
}
|
||||
|
||||
fn unavailable(mut self, reason: &'static str) -> Self {
|
||||
self.reason = reason;
|
||||
self.availability = true;
|
||||
self
|
||||
}
|
||||
|
||||
fn undecoded() -> Self {
|
||||
Self {
|
||||
reason: "invalid",
|
||||
@@ -47,6 +61,7 @@ impl RuntimeRequestProofError {
|
||||
exp: None,
|
||||
now: None,
|
||||
time_delta_seconds: None,
|
||||
availability: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +121,18 @@ impl RuntimeRequestProofError {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::Error> for RuntimeRequestProofError {
|
||||
fn from(error: crate::Error) -> Self {
|
||||
let reason = match error {
|
||||
crate::Error::DbCapacityExceeded => "db_capacity_exceeded",
|
||||
crate::Error::DbExecutorUnavailable => "db_executor_unavailable",
|
||||
crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown",
|
||||
_ => "database_authority_unavailable",
|
||||
};
|
||||
Self::undecoded().unavailable(reason)
|
||||
}
|
||||
}
|
||||
|
||||
/// Domain-separated fixed-length fingerprints, not raw request-derived IDs.
|
||||
/// Operators can correlate a known ID by applying this same function.
|
||||
pub fn diagnostic_id_hash(id: &str) -> String {
|
||||
@@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof(
|
||||
body_digest: &str,
|
||||
) -> Result<VerifiedRuntimeRequestSource, RuntimeRequestProofError> {
|
||||
verify_runtime_request_source_proof_with_store(
|
||||
api.store.as_ref(),
|
||||
Arc::clone(&api.store),
|
||||
&api.config,
|
||||
proof,
|
||||
workspace_id,
|
||||
@@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn verify_runtime_request_source_proof_with_store(
|
||||
store: &dyn ControlPlaneStore,
|
||||
store: Arc<dyn ControlPlaneStore>,
|
||||
config: &ServerConfig,
|
||||
proof: &str,
|
||||
workspace_id: &str,
|
||||
@@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
store: &dyn ControlPlaneStore,
|
||||
store: Arc<dyn ControlPlaneStore>,
|
||||
config: &ServerConfig,
|
||||
proof: &str,
|
||||
workspace_id: &str,
|
||||
@@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
let unverified = decode_runtime_request_source_claims(proof)
|
||||
.map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?;
|
||||
let diagnostic = RuntimeRequestProofError::decoded(&unverified);
|
||||
let audience = remote_audience(config, workspace_id)
|
||||
.map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?;
|
||||
let audience = remote_audience(config, workspace_id).map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.unavailable("backend_audience_unavailable")
|
||||
})?;
|
||||
let trusted = store
|
||||
.get_workspace_runtime_binding(workspace_id, &unverified.iss)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("runtime_trust_authority_unavailable")
|
||||
.unavailable("runtime_trust_authority_unavailable")
|
||||
})?
|
||||
.filter(|record| record.revoked_at.is_none())
|
||||
.ok_or_else(|| {
|
||||
@@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
&consumed_at,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))?
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.unavailable("replay_authority_unavailable")
|
||||
})?
|
||||
{
|
||||
return Err(diagnostic.reason("replay"));
|
||||
}
|
||||
@@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
runtime_id: claims.iss.clone(),
|
||||
worker_id: worker_id.to_owned(),
|
||||
};
|
||||
let member = store
|
||||
.get_worker_registry(workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("worker_catalog_authority_unavailable")
|
||||
})?;
|
||||
let reserved = store
|
||||
.has_active_worker_create_reservation(workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("worker_reservation_authority_unavailable")
|
||||
})?;
|
||||
if member.is_none() && !reserved {
|
||||
return Err(diagnostic.reason("worker_catalog_membership"));
|
||||
}
|
||||
store
|
||||
.require_current_worker_singleton_owner(workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("worker_singleton_authority_denied")
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let executor = store.db_executor();
|
||||
let stage_diagnostic = diagnostic.clone();
|
||||
// Only synchronous catalog authority lives in this closure. The trust
|
||||
// and JTI async wrappers above already offload through the executor.
|
||||
executor
|
||||
.run(move || {
|
||||
let member = store
|
||||
.get_worker_registry(&workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.unavailable("worker_catalog_authority_unavailable")
|
||||
})?;
|
||||
let reserved = store
|
||||
.has_active_worker_create_reservation(&workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.unavailable("worker_reservation_authority_unavailable")
|
||||
})?;
|
||||
if member.is_none() && !reserved {
|
||||
return Err(stage_diagnostic.reason("worker_catalog_membership"));
|
||||
}
|
||||
store
|
||||
.require_current_worker_singleton_owner(&workspace_id, &worker)
|
||||
.map_err(|error| {
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbCapacityExceeded
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::Sqlite(_)
|
||||
) {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.unavailable("worker_singleton_authority_unavailable")
|
||||
} else {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.reason("worker_singleton_authority_denied")
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|error: RuntimeRequestProofError| {
|
||||
if error.verification == "unverified" && error.is_availability_error() {
|
||||
diagnostic.clone().unavailable(error.reason)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
Ok(VerifiedRuntimeRequestSource {
|
||||
@@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
authentication_binding_id: trusted.binding_id,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::{
|
||||
AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode,
|
||||
WorkspaceRuntimeBinding,
|
||||
};
|
||||
use std::time::Duration;
|
||||
use worker_runtime::auth::{
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner,
|
||||
request_body_digest,
|
||||
};
|
||||
|
||||
const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa";
|
||||
const NOW: i64 = 1_700_000_000;
|
||||
const PATH: &str = "/api/runtime/v1/resources/fetch";
|
||||
|
||||
// rusqlite's busy handler accepts a function pointer, not a closure. Each
|
||||
// fixture owns a dedicated single-worker executor, so this thread-local
|
||||
// handshake is isolated from all parallel tests and product connections.
|
||||
thread_local! {
|
||||
static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell<Option<(
|
||||
tokio::sync::oneshot::Sender<()>,
|
||||
std::sync::mpsc::Receiver<()>,
|
||||
)>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn acknowledge_sqlite_contention(_attempt: i32) -> bool {
|
||||
SQLITE_BUSY_HANDSHAKE.with(|slot| {
|
||||
let Some((blocked, retry)) = slot.borrow_mut().take() else {
|
||||
return false;
|
||||
};
|
||||
let _ = blocked.send(());
|
||||
// The test releases SQLite's writer lock before allowing retry.
|
||||
// Dropping the sender on panic also releases this worker promptly.
|
||||
retry.recv_timeout(Duration::from_secs(3)).is_ok()
|
||||
})
|
||||
}
|
||||
|
||||
async fn fixture(
|
||||
root: &std::path::Path,
|
||||
) -> (Arc<SqliteWorkspaceStore>, ServerConfig, String, String) {
|
||||
let db_path = root.join("auth.db");
|
||||
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open_for_test(&db_path)
|
||||
.unwrap()
|
||||
.with_db_executor(executor),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-auth".into(),
|
||||
kind: "user".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
let workspace = WorkspaceRecord {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
owner_account_id: "account-auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
};
|
||||
store.upsert_workspace(&workspace).await.unwrap();
|
||||
let config = ServerConfig {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
workspace_display_name: "Auth".into(),
|
||||
workspace_created_at: "1".into(),
|
||||
workspace_execution_root: root.to_owned(),
|
||||
database_path: db_path,
|
||||
data_root: root.to_owned(),
|
||||
embedded_runtime_store_root: root.join("runtime"),
|
||||
static_assets_dir: None,
|
||||
auth: crate::server::AuthConfig::Passkey {
|
||||
rp_id: "localhost".into(),
|
||||
origin: "http://localhost".into(),
|
||||
public_base_url: "http://localhost".into(),
|
||||
cookie_name: "session".into(),
|
||||
},
|
||||
max_records: 10,
|
||||
repositories: vec![],
|
||||
runtime_event_sources: vec![],
|
||||
remote_runtime_sources: vec![],
|
||||
backend_base_url: Some("server-auth".into()),
|
||||
};
|
||||
let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap();
|
||||
store
|
||||
.upsert_workspace_runtime_binding_record(
|
||||
WorkspaceRuntimeBinding {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
runtime_id: "runtime-auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
base_url: "http://127.0.0.1:1".into(),
|
||||
public_key: identity.public_key.clone(),
|
||||
public_key_fingerprint: String::new(),
|
||||
binding_id: String::new(),
|
||||
authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
revoked_at: None,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let binding = ControlPlaneStore::get_workspace_runtime_binding(
|
||||
store.as_ref(),
|
||||
WORKSPACE,
|
||||
"runtime-auth",
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let proof = RuntimeRequestSourceSigner::from_identity(&identity)
|
||||
.issue(
|
||||
"server-auth",
|
||||
WORKSPACE,
|
||||
None,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
b"{}",
|
||||
NOW,
|
||||
30,
|
||||
)
|
||||
.unwrap();
|
||||
(store, config, proof, binding.binding_id)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (store, config, proof, binding_id) = fixture(root.path()).await;
|
||||
// A real independent SQLite writer prevents JTI insertion while trust
|
||||
// reads remain possible. Rollback on Drop also releases this on panic.
|
||||
let mut writer = rusqlite::Connection::open(&config.database_path).unwrap();
|
||||
let lock = writer
|
||||
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let (blocked, contention_reached) = tokio::sync::oneshot::channel();
|
||||
let (retry, retry_rx) = std::sync::mpsc::channel();
|
||||
let setup_store = store.clone();
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || -> crate::Result<()> {
|
||||
SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx)));
|
||||
setup_store.with_conn(|conn| {
|
||||
conn.busy_handler(Some(acknowledge_sqlite_contention))?;
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let request_store = store.clone();
|
||||
let request_config = config.clone();
|
||||
let request_proof = proof.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
verify_runtime_request_source_proof_with_clock(
|
||||
request_store,
|
||||
&request_config,
|
||||
&request_proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), contention_reached)
|
||||
.await
|
||||
.expect(
|
||||
"SQLite JTI persistence must report real writer contention without blocking Tokio",
|
||||
)
|
||||
.unwrap();
|
||||
// The busy callback acknowledges an actual SQLite write blocked by the
|
||||
// independent writer, then holds retry until explicitly released.
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"JTI must remain pending while SQLite writer is locked"
|
||||
);
|
||||
let control = tokio::spawn(async { "control-responsive" });
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), control)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
"control-responsive"
|
||||
);
|
||||
// This timer is the behavior being tested, not a guessed wait for SQL.
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
tokio::time::sleep(Duration::from_millis(1)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"timer must advance while JTI remains blocked"
|
||||
);
|
||||
lock.rollback().unwrap();
|
||||
retry.send(()).unwrap();
|
||||
let source = tokio::time::timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(source.runtime_id, "runtime-auth");
|
||||
assert_eq!(source.authentication_binding_id, binding_id);
|
||||
let replay = verify_runtime_request_source_proof_with_clock(
|
||||
store,
|
||||
&config,
|
||||
&proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(replay.reason, "replay");
|
||||
assert!(!replay.is_availability_error());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_database_is_availability_failure_not_invalid_proof() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (store, config, proof, _) = fixture(root.path()).await;
|
||||
// Fill the isolated executor with a blocked job and queued jobs. No
|
||||
// scheduling sleeps: the worker explicitly acknowledges its start.
|
||||
let executor = store.db_executor();
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker_executor = executor.clone();
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_executor
|
||||
.run(move || -> crate::Result<()> {
|
||||
let _ = started.send(());
|
||||
release_rx.recv().unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) }));
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
verify_runtime_request_source_proof_with_clock(
|
||||
store,
|
||||
&config,
|
||||
&proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
// Release before assertions so a failed contract never strands a worker.
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap().unwrap();
|
||||
queued.await.unwrap();
|
||||
let error = outcome
|
||||
.expect("capacity rejection must not wait for queued SQL")
|
||||
.unwrap_err();
|
||||
assert_eq!(error.reason, "runtime_trust_authority_unavailable");
|
||||
assert!(error.is_availability_error());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_is_atomic_and_exact_retries_converge() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
Arc::new(
|
||||
@@ -339,7 +339,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn create_rejects_noncanonical_repository_keys_without_persisting_a_workspace() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
Arc::new(
|
||||
@@ -378,7 +378,7 @@ mod tests {
|
||||
for with_repository in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let database_path = temp.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
@@ -418,7 +418,7 @@ mod tests {
|
||||
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let restarted = WorkspaceCatalogService::new(store.clone(), materials);
|
||||
let created = restarted.create(request, owner_account_id).unwrap();
|
||||
let identity = store
|
||||
@@ -435,7 +435,7 @@ mod tests {
|
||||
for with_repository in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let database_path = temp.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default());
|
||||
let service = WorkspaceCatalogService::new(store.clone(), materials.clone());
|
||||
@@ -486,7 +486,7 @@ mod tests {
|
||||
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let restarted = WorkspaceCatalogService::new(store.clone(), materials);
|
||||
let created = restarted.create(request, owner_account_id).unwrap();
|
||||
let identity = store
|
||||
@@ -500,7 +500,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn idempotency_key_reuse_with_different_payload_is_rejected() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store,
|
||||
@@ -532,7 +532,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn repository_free_create_initializes_authority_and_concurrent_retries_converge() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
@@ -611,7 +611,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replay_reads_current_repository_at_initial_key_and_never_recreates_it() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
@@ -673,13 +673,13 @@ mod tests {
|
||||
repository: None,
|
||||
};
|
||||
let (owner, created) = {
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
|
||||
let owner = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(store, materials.clone());
|
||||
let created = service.create(request.clone(), owner.clone()).unwrap();
|
||||
(owner, created)
|
||||
};
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
|
||||
let reopened = WorkspaceCatalogService::new(store.clone(), materials);
|
||||
let replay = reopened.create(request, owner).unwrap();
|
||||
assert_eq!(replay.workspace, created.workspace);
|
||||
@@ -739,7 +739,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn create_rejects_non_user_account_owners() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "organization-owner".to_string(),
|
||||
@@ -777,7 +777,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn catalog_list_is_scoped_to_the_required_owner_account() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner_a = owner_account(store.as_ref());
|
||||
let owner_b = "account-owner-b".to_string();
|
||||
store
|
||||
@@ -845,7 +845,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn remote_repository_creation_persists_typed_source_without_auth_metadata() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
|
||||
@@ -236,7 +236,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
"Workspace deletion request exceeds bounded field limits".to_string(),
|
||||
));
|
||||
}
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
if let Some(existing) = read_operation(tx, &request.operation_id)? {
|
||||
if existing.actor_account_id != actor_account_id {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
@@ -369,7 +369,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
child_operation_id: &str,
|
||||
) -> Result<WorkspaceDeletionOperationResponse> {
|
||||
validate_operation_id(child_operation_id)?;
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
let operation = read_operation(tx, operation_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?
|
||||
.response;
|
||||
@@ -411,7 +411,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
failure_category: Option<&str>,
|
||||
) -> Result<WorkspaceDeletionOperationResponse> {
|
||||
validate_operation_projection(child_operation_ids, blockers)?;
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
let current = read_operation(tx, operation_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?
|
||||
.response;
|
||||
@@ -462,7 +462,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
&self,
|
||||
operation_id: &str,
|
||||
) -> Result<WorkspaceDeletionOperationResponse> {
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?;
|
||||
let operation = read_operation(tx, operation_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?;
|
||||
@@ -922,7 +922,7 @@ mod tests {
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn setup() -> (SqliteWorkspaceStore, String, String) {
|
||||
seed_workspaces(SqliteWorkspaceStore::in_memory().expect("store"))
|
||||
seed_workspaces(SqliteWorkspaceStore::in_memory_for_test().expect("store"))
|
||||
}
|
||||
|
||||
fn seed_workspaces(store: SqliteWorkspaceStore) -> (SqliteWorkspaceStore, String, String) {
|
||||
@@ -989,7 +989,7 @@ mod tests {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let path = dir.path().join("server.db");
|
||||
let (store, owner, workspace_id) =
|
||||
seed_workspaces(SqliteWorkspaceStore::open(&path).expect("store"));
|
||||
seed_workspaces(SqliteWorkspaceStore::open_for_test(&path).expect("store"));
|
||||
for workspace in ["workspace-a", "workspace-b"] {
|
||||
let backend = ticket::SqliteTicketBackend::open_verified(&path, workspace).unwrap();
|
||||
let initial = backend
|
||||
@@ -1019,7 +1019,7 @@ mod tests {
|
||||
// Seed domain handling evidence under a transaction. This fixture
|
||||
// exercises owner purge, not singleton authentication or delivery.
|
||||
store
|
||||
.with_transaction(|tx| {
|
||||
.with_immediate_transaction(|tx| {
|
||||
ticket::sqlite_update_ticket_assignment_tracking_in_transaction(
|
||||
tx,
|
||||
workspace,
|
||||
|
||||
@@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost {
|
||||
impl WorkspaceDriveHost {
|
||||
pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result<Self> {
|
||||
let storage = storage.clone();
|
||||
let registration = tokio::task::spawn_blocking(move || Drive::register(&storage))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))?
|
||||
.map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?;
|
||||
let registration = crate::db_executor::DbExecutor::shared()
|
||||
.run(move || {
|
||||
Drive::register(&storage).map_err(|error| {
|
||||
Error::Store(format!("failed to register Drive storage: {error}"))
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
Ok(Self {
|
||||
registration,
|
||||
purged: AtomicBool::new(false),
|
||||
@@ -80,21 +83,21 @@ impl WorkspaceDriveHost {
|
||||
}
|
||||
let storage = storage.clone();
|
||||
let registration = self.registration.clone();
|
||||
let drive = tokio::task::spawn_blocking(move || {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain IMMEDIATE transactions lock the attached server authority
|
||||
// as well as Drive metadata. Deletion reservation and cached-handle
|
||||
// operations therefore serialize with other server DB writes.
|
||||
Drive::open_with_workspace_authority(
|
||||
&storage,
|
||||
®istration,
|
||||
&root,
|
||||
&authority_database,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??;
|
||||
let drive = crate::db_executor::DbExecutor::shared()
|
||||
.run(move || {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain IMMEDIATE transactions lock the attached server authority
|
||||
// as well as Drive metadata. Deletion reservation and cached-handle
|
||||
// operations therefore serialize with other server DB writes.
|
||||
Drive::open_with_workspace_authority(
|
||||
&storage,
|
||||
®istration,
|
||||
&root,
|
||||
&authority_database,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await?;
|
||||
*cached = Some(drive.clone());
|
||||
Ok(drive)
|
||||
}
|
||||
@@ -109,9 +112,9 @@ impl WorkspaceDriveHost {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root, authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from))
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || drive.fence().map_err(Error::from))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))?
|
||||
}
|
||||
|
||||
pub(crate) async fn purge(
|
||||
@@ -124,20 +127,20 @@ impl WorkspaceDriveHost {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root.clone(), authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
|
||||
drive.purge()?;
|
||||
// Remove the empty blob root, never the data root or another Workspace.
|
||||
// Missing roots are successful retries; other failures retain FeatureStorage.
|
||||
match std::fs::remove_dir_all(&root) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??;
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || -> Result<()> {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
|
||||
drive.purge()?;
|
||||
// Remove the empty blob root, never the data root or another Workspace.
|
||||
// Missing roots are successful retries; other failures retain FeatureStorage.
|
||||
match std::fs::remove_dir_all(&root) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
// A later signing-material/finalization failure can retry after the
|
||||
// feature DB has been closed and removed by FeatureStorage.delete.
|
||||
self.purged.store(true, Ordering::Release);
|
||||
|
||||
@@ -780,7 +780,7 @@ mod tests {
|
||||
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let database_path = temp.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-1".to_string(),
|
||||
@@ -833,7 +833,7 @@ mod tests {
|
||||
|
||||
drop(failing_service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let service = WorkspaceSigningIdentityService::new(store.clone(), materials.clone());
|
||||
let provisioned = service
|
||||
.provision_existing("workspace-1", "account-1")
|
||||
@@ -962,7 +962,7 @@ mod tests {
|
||||
);
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let restarted = WorkspaceSigningIdentityService::new(store.clone(), materials.clone());
|
||||
let recovered = restarted
|
||||
.provision_existing("workspace-2", "account-1")
|
||||
|
||||
@@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription<S>(
|
||||
worker_id,
|
||||
runtime_id: Some(runtime_id),
|
||||
} => {
|
||||
let connection = match WorkspaceWorker::resolve(
|
||||
let connection = match WorkspaceWorker::resolve_async(
|
||||
&api,
|
||||
&runtime_id,
|
||||
worker_id.as_str(),
|
||||
) {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(worker) => worker
|
||||
.connect_protocol(&context)
|
||||
.await
|
||||
@@ -366,7 +368,8 @@ async fn run_workspace_workers(
|
||||
) {
|
||||
let Ok(mut projection) = api
|
||||
.worker_projection
|
||||
.subscribe_ordered(api.config.max_records.max(1))
|
||||
.subscribe_ordered_async(api.config.max_records.max(1))
|
||||
.await
|
||||
else {
|
||||
let _ = send_rejected(
|
||||
&outbound,
|
||||
@@ -377,11 +380,22 @@ async fn run_workspace_workers(
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let mut workers = projection
|
||||
.take_snapshot()
|
||||
.into_iter()
|
||||
.filter_map(|record| project_registry_worker(&api, record))
|
||||
.collect::<Vec<_>>();
|
||||
let Ok(mut workers) = project_registry_workers_async(
|
||||
api.store.clone(),
|
||||
api.config.workspace_id.clone(),
|
||||
projection.take_snapshot(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
let _ = send_rejected(
|
||||
&outbound,
|
||||
request_id,
|
||||
SubscriptionRejectionCode::Internal,
|
||||
"failed to enrich durable Worker projection".to_string(),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
sort_workers(&mut workers);
|
||||
if send_frame(
|
||||
&outbound,
|
||||
@@ -406,7 +420,29 @@ async fn run_workspace_workers(
|
||||
for change in event.changes {
|
||||
let payload = match change {
|
||||
crate::worker_projection::WorkerProjectionChange::Upsert(record) => {
|
||||
let Some(worker) = project_registry_worker(&api, record) else {
|
||||
let workers = project_registry_workers_async(
|
||||
api.store.clone(),
|
||||
api.config.workspace_id.clone(),
|
||||
vec![record],
|
||||
)
|
||||
.await;
|
||||
let Ok(mut workers) = workers else {
|
||||
// Admission failure cannot silently lose a live
|
||||
// update: terminate and require a fresh snapshot.
|
||||
let _ = send_frame(
|
||||
&outbound,
|
||||
SubscriptionFrame::new(SubscriptionFramePayload::Event(
|
||||
SubscriptionEvent::SubscriptionClosed {
|
||||
subscription_id: subscription_id.clone(),
|
||||
code: SubscriptionTerminationCode::Lagged,
|
||||
message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(),
|
||||
},
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let Some(worker) = workers.pop() else {
|
||||
continue;
|
||||
};
|
||||
SubscriptionEventPayload::WorkerUpserted { worker }
|
||||
@@ -450,8 +486,28 @@ async fn run_workspace_workers(
|
||||
}
|
||||
}
|
||||
|
||||
async fn project_registry_workers_async(
|
||||
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
|
||||
workspace_id: String,
|
||||
records: Vec<crate::store::WorkerRegistryProjectionRecord>,
|
||||
) -> crate::Result<Vec<SubscriptionWorker>> {
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
// Keep the committed Worker payload; only the existing Workspace-scoped
|
||||
// workdir/history/repository enrichment reads occur inside this job.
|
||||
Ok(records
|
||||
.into_iter()
|
||||
.filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record))
|
||||
.collect())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
// Synchronous helpers stay synchronous; they run only inside the job above.
|
||||
fn project_registry_worker(
|
||||
api: &WorkspaceApi,
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
record: crate::store::WorkerRegistryProjectionRecord,
|
||||
) -> Option<SubscriptionWorker> {
|
||||
let runtime_id = record.registry.worker.runtime_id.clone();
|
||||
@@ -491,29 +547,25 @@ fn project_registry_worker(
|
||||
attempt_id: job.attempt_id,
|
||||
purpose: job.purpose,
|
||||
});
|
||||
if !project_working_directory(api, &record.registry.worker, &mut worker) {
|
||||
if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) {
|
||||
return None;
|
||||
}
|
||||
Some(worker)
|
||||
}
|
||||
|
||||
fn project_working_directory(
|
||||
api: &WorkspaceApi,
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
worker_ref: &RuntimeWorkerRef,
|
||||
worker: &mut SubscriptionWorker,
|
||||
) -> bool {
|
||||
let Ok(links) = api
|
||||
.store
|
||||
.list_worker_workdir_links(&api.config.workspace_id, worker_ref)
|
||||
else {
|
||||
let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else {
|
||||
return false;
|
||||
};
|
||||
if !links.is_empty() {
|
||||
let mut attachments = Vec::with_capacity(links.len());
|
||||
for link in links {
|
||||
let Ok(Some(workdir)) = api
|
||||
.store
|
||||
.get_workdir_registry(&api.config.workspace_id, &link.workdir_id)
|
||||
let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -533,9 +585,8 @@ fn project_working_directory(
|
||||
}
|
||||
worker.workdir_attachments = attachments;
|
||||
} else {
|
||||
let Ok(has_link_history) = api
|
||||
.store
|
||||
.worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref)
|
||||
let Ok(has_link_history) =
|
||||
store.worker_workdir_link_history_exists(workspace_id, worker_ref)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -543,13 +594,17 @@ fn project_working_directory(
|
||||
worker.workdir_attachments.clear();
|
||||
}
|
||||
}
|
||||
project_repository_keys(api, worker)
|
||||
project_repository_keys(store, workspace_id, worker)
|
||||
}
|
||||
|
||||
fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool {
|
||||
fn project_repository_keys(
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
worker: &mut SubscriptionWorker,
|
||||
) -> bool {
|
||||
project_repository_keys_with(worker, |repository_id| {
|
||||
api.store
|
||||
.get_repository(&api.config.workspace_id, repository_id)
|
||||
store
|
||||
.get_repository(workspace_id, repository_id)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|repository| repository.repository_key)
|
||||
@@ -634,9 +689,8 @@ mod tests {
|
||||
protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||
let mut worker = SubscriptionWorker {
|
||||
fn external_worker() -> SubscriptionWorker {
|
||||
SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: worker_id(),
|
||||
runtime_id: Some("runtime-1".to_string()),
|
||||
@@ -660,8 +714,12 @@ mod tests {
|
||||
.unwrap(),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||
let mut worker = external_worker();
|
||||
assert!(project_repository_keys_with(&mut worker, |_| {
|
||||
panic!("External Workdir must not require Repository projection")
|
||||
}));
|
||||
@@ -673,6 +731,110 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository()
|
||||
{
|
||||
use crate::store::{
|
||||
AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord,
|
||||
WorkspaceRecord,
|
||||
};
|
||||
let store = std::sync::Arc::new(
|
||||
SqliteWorkspaceStore::in_memory_for_test()
|
||||
.unwrap()
|
||||
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".into(),
|
||||
kind: "user".into(),
|
||||
handle: "owner".into(),
|
||||
display_name: "Owner".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-1".into(),
|
||||
owner_account_id: "owner".into(),
|
||||
display_name: "Enrichment".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1");
|
||||
store
|
||||
.upsert_worker_registry(&WorkerRegistryRecord {
|
||||
workspace_id: "workspace-1".into(),
|
||||
worker: worker_ref.clone(),
|
||||
display_name: "Durable name".into(),
|
||||
profile: Some("builtin:coder".into()),
|
||||
retention_state: "normal".into(),
|
||||
transcript_ref: None,
|
||||
session_ref: None,
|
||||
summary_ref: None,
|
||||
diagnostics_ref: None,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2")
|
||||
.unwrap();
|
||||
let committed = store
|
||||
.worker_registry_projection("workspace-1", &worker_ref)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
// Change authority after capturing the payload, as can happen while async
|
||||
// enrichment is queued. It must not re-query and replace the committed Worker.
|
||||
let mut later = external_worker();
|
||||
later.state = protocol::subscription::SubscriptionWorkerState::Running;
|
||||
store
|
||||
.apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3")
|
||||
.unwrap();
|
||||
let projected = project_registry_workers_async(
|
||||
store.clone(),
|
||||
"workspace-1".into(),
|
||||
vec![committed.clone()],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(projected.len(), 1);
|
||||
let worker = &projected[0];
|
||||
assert_eq!(
|
||||
worker.state,
|
||||
protocol::subscription::SubscriptionWorkerState::Idle
|
||||
);
|
||||
assert_eq!(
|
||||
worker.availability,
|
||||
protocol::subscription::SubscriptionWorkerAvailability::Observed
|
||||
);
|
||||
assert_eq!(worker.display_name.as_deref(), Some("Durable name"));
|
||||
assert_eq!(worker.profile.as_deref(), Some("builtin:coder"));
|
||||
assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1"));
|
||||
assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1"));
|
||||
assert_eq!(
|
||||
worker.workdir_attachments,
|
||||
external_worker().workdir_attachments
|
||||
);
|
||||
let mut unresolved = committed;
|
||||
unresolved
|
||||
.observation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.worker
|
||||
.workdir_attachments[0]
|
||||
.repository_id = Some("missing-repository".into());
|
||||
assert!(
|
||||
project_registry_workers_async(store, "workspace-1".into(), vec![unresolved])
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_with_events(events: mpsc::Receiver<protocol::Event>) -> SubscriptionResponse {
|
||||
let (control_outbound, mut control_receiver) = mpsc::channel(1);
|
||||
let (protocol_outbound, _protocol_receiver) = mpsc::channel(1);
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# Backend bounded synchronous execution
|
||||
|
||||
## Ownership and admission
|
||||
|
||||
`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner.
|
||||
Use `executor.run(move || synchronous_operation(...)).await` with owned input and
|
||||
services. The closure owns the **whole** synchronous operation, including connection
|
||||
acquisition, mutex wait, the existing SQL transaction, commit, and ordered local
|
||||
publication. Do not split source authorization, receipts, CAS, audit, or saved
|
||||
results around an await. Do not change connection ownership or introduce a read pool.
|
||||
|
||||
Production stores share one DB worker and a queue of 32 jobs. Admission uses a
|
||||
nonblocking queue send and a separate actual-job budget (worker count plus queue
|
||||
capacity); rejection means the operation was **not accepted**. HTTP maps capacity
|
||||
and executor unavailability to 503, not authentication failure or deletion conflict.
|
||||
Tests can inject a dedicated executor with explicit limits, without process-global
|
||||
settings or environment variables. Ordinary SQLite fixtures use `open_for_test`,
|
||||
`in_memory_for_test`, or `from_connection_for_test`: their clones share their own
|
||||
1-worker/32-queued owner, not the production singleton or another fixture's drain.
|
||||
Server/Workspace fixture constructors likewise inject a lazy `HostExecutors` owner;
|
||||
all child Workspaces and weak retention adapters use that same owner. Host and
|
||||
cleanup still have the production 2-worker/16-queued limits. Dropping the final
|
||||
sender closes its queue after accepted work finishes; idle fixture threads are
|
||||
not kept in a process-global fixture cache.
|
||||
|
||||
The restore intent journal and Workspace deletion use
|
||||
`with_immediate_transaction`: read authority/journal state only after acquiring
|
||||
SQLite write ownership. A DEFERRED read-to-write upgrade can return BUSY without
|
||||
invoking SQLite's busy handler when another connection is writing. Moving these
|
||||
whole helpers to an offloaded caller must not create an automatic mutation retry.
|
||||
|
||||
|
||||
Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter:
|
||||
|
||||
- does not interrupt, resubmit, or retry SQL;
|
||||
- does not return its queue/job slot before execution and discarded-result cleanup;
|
||||
- does not release the request lifecycle lease before actual completion.
|
||||
|
||||
Synchronous post-commit continuations inherit the admitted lifecycle source when
|
||||
submitting another bounded job. This is not implicit inheritance by detached Tokio
|
||||
tasks. An operation panic has an unknown outcome even if SQL already committed;
|
||||
return a structured unknown-outcome error and diagnostic, never retry the mutation.
|
||||
A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all
|
||||
jobs accepted before its barrier is first polled and waits for actual cleanup; it
|
||||
is not a queue close and does not include later admissions. Lifecycle callers must
|
||||
stop their producers/admission before destroying resources.
|
||||
|
||||
## Runtime/network boundaries
|
||||
|
||||
A synchronous Runtime workflow must not monopolize the sole DB worker while
|
||||
waiting for callbacks that themselves need DB authority. The Server's separate
|
||||
finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job
|
||||
ownership mechanism, not an unbounded `spawn_blocking` queue.
|
||||
|
||||
Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can
|
||||
use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an
|
||||
owned workflow needs async resource locks, `run_host_workflow` polls it on a Host
|
||||
worker using the original Tokio handle; do not build a replacement runtime for
|
||||
cached services and do not submit to and await the Host lane from inside itself.
|
||||
Control/session guards needed by an accepted mutation belong to that actual job.
|
||||
|
||||
Backend Job cleanup has a third finite lane (two workers, 16 queued jobs), so a
|
||||
long Run teardown wait cannot starve ordinary Host control/stop requests. It is
|
||||
admitted before its durable cleanup claim. Rejection leaves Pending/Failed
|
||||
ownership recoverable instead of abandoning an Executing claim. Ticket
|
||||
checker dispatch and timeout processing use bounded admission; overload leaves the
|
||||
saved reservation/deadline diagnosable. They are not automatically retried after an
|
||||
unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof
|
||||
of model processing.
|
||||
|
||||
Remote subscription and Workdir authorization are async callbacks: their complete
|
||||
live binding check and fresh request signing are offloaded together. A queue error
|
||||
must not fall back to a configured bearer token. Synchronous CLI/migration callers
|
||||
and existing synchronous Runtime adapters retain their explicit blocking boundary.
|
||||
|
||||
## Caller recipe (including assignment APIs)
|
||||
|
||||
```rust
|
||||
let store = api.store.clone();
|
||||
let workspace_id = api.workspace_id().to_owned();
|
||||
let worker = worker.clone();
|
||||
let record = api.store.db_executor().run(move || {
|
||||
store.get_worker_registry(&workspace_id, &worker)
|
||||
}).await?;
|
||||
```
|
||||
|
||||
This is an illustration, not a new authority API. Assignment delivery scan,
|
||||
reserve, prepare, acknowledgement and start authorization must keep their existing
|
||||
whole transaction helpers. Additional assignment diagnostics, answer mutations,
|
||||
and Ticket queries use the same owned caller boundary; do not add a second ledger
|
||||
or reconstruct processing evidence from Worker liveness.
|
||||
|
||||
## Regression evidence
|
||||
|
||||
- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread
|
||||
timer/control task progresses; accepted/cancelled queue bounds; rollback; panic
|
||||
before/after commit; discarded reply cleanup and retained authority; child jobs.
|
||||
- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope
|
||||
opening/mutation under a separate SQLite connection's write lock; overload;
|
||||
cancelled HTTP jobs and Workspace deletion drain; same-budget fixture isolation,
|
||||
saturated/cancelled Host jobs versus cleanup and paused Tokio time; restore intent
|
||||
contention before the first authority read, with one persisted tuple and no retry.
|
||||
The terminal checker fixture keeps its original Tokio driver alive through accepted cleanup,
|
||||
then shuts down projection producers and drains actual DB jobs.
|
||||
- `worker_projection::tests`: real SQL contention, commit/payload publication,
|
||||
ordered snapshot registration, and replacement draining a cancelled consumer's
|
||||
accepted SQL before applying a new Runtime snapshot.
|
||||
- Auth/proof and Runtime subscription tests protect availability classification,
|
||||
replay rejection, and no bearer fallback.
|
||||
- `workdir`'s `http-client` tests protect awaited fresh authorization and exact
|
||||
request claims before any HTTP send.
|
||||
|
||||
These fixtures use temporary authority data and observable synchronization, not
|
||||
live databases or a fixed sleep to guess completion. This change does not claim a
|
||||
latency benchmark or a measured production incident.
|
||||
@@ -0,0 +1,166 @@
|
||||
# T-748: default-parallel suite failure diagnosis
|
||||
|
||||
## Sources and preserved observations
|
||||
|
||||
- Original local candidate: `61533fadb6b39465af60cb55aed2b4d0650319cd`.
|
||||
- Comparison source: `56795fb92cb9def9bcd701bd51d92e99526f8eff` (develop/T742).
|
||||
- Same implementation branch: `work/T-748-bounded-db-execution`.
|
||||
- The original `target/t748-server-tests.log` remains untouched. Its 97 FAILED
|
||||
lines and 600-second timeout were symptoms, not proof of one cause.
|
||||
- Initial diagnosis used `cargo test -p yoi-workspace-server -- --nocapture`,
|
||||
default libtest parallelism, without changing queue limits, features, or startup
|
||||
settings. `target/t748-diagnostic-environment.txt` records the environment;
|
||||
this host reports 24 available CPUs. Diagnostic captures were bounded to
|
||||
90/150 seconds to obtain errors before the original timeout, not counted as
|
||||
successful full-suite runs.
|
||||
|
||||
Logs under `target/` are local diagnostic artifacts, not tracked review inputs.
|
||||
This report records their meaning and the permanent regressions that reproduce it.
|
||||
|
||||
## 1. Different fixtures consumed one global Host admission budget
|
||||
|
||||
The first non-intentional errors in `target/t748-server-nocapture.log` were
|
||||
`DbCapacityExceeded` in manual Worker setup (`server.rs` at the then-current
|
||||
line 50614), followed by assignment fixture construction (52990/53045) and
|
||||
HTTP 503 responses. The deliberate before/after-commit executor panic tests
|
||||
are different: those passed and verified unknown-outcome diagnostics.
|
||||
|
||||
Temporary submit/start/cleanup instrumentation recorded submitting test names,
|
||||
closure types, actual job IDs and the execution-owner identity. It showed a
|
||||
single Host lane with `max=18`, with accepted jobs from unrelated fixtures.
|
||||
A rejection snapshot contained `{32,45,46,...,61}`. The complete diagnostic
|
||||
capture later contained cleanup for every submitted Host job on that owner;
|
||||
it did not show permanently abandoned accepted Host jobs. Instrumentation was
|
||||
removed before the source was committed/published.
|
||||
|
||||
`target/t748-live-process-18.txt` sampled the test process and threads at the
|
||||
actual rejection. That sample had 126 threads; the Host workers were runnable,
|
||||
and the process was using approximately 17 CPU cores. There was no captured
|
||||
thread-creation EAGAIN/unavailable error. This evidence does not establish an
|
||||
OS thread/CPU hard limit as the cause. No test processes remained after the
|
||||
bounded diagnostic run or the completed suite.
|
||||
|
||||
A representative failure passed alone (0.56 seconds); the eight logging cases
|
||||
passed with four test threads (2.46 seconds). The isolated comparison source's
|
||||
full default-parallel suite passed: 931 library tests, one pre-existing ignored
|
||||
migration fixture, 11 binary tests; library duration 168.74 seconds. This did
|
||||
not prove that all new failures were capacity-related, but did distinguish the
|
||||
new shared-admission rejection from an observed baseline failure.
|
||||
|
||||
### Ownership fix (`f3d4aba5`)
|
||||
|
||||
Production budgets remain shared and finite: DB 1 worker/32 queued, Host 2/16,
|
||||
cleanup 2/16. Ordinary fixtures now explicitly own a finite DB executor and lazy
|
||||
Host/cleanup owners at those same limits. API clones, child Workspace APIs, and
|
||||
weak retention adapters use their injected owner. No global fixture lane cache,
|
||||
parallelism reduction, unlimited queue, or capacity-wait retry was added.
|
||||
|
||||
The permanent saturated-Host regression fills both running slots and all 16
|
||||
queued slots, cancels waiters, advances paused Tokio time, and still observes
|
||||
rejection until actual completion. Another fixture and the separate cleanup lane
|
||||
can progress; draining restores the original budget and completes exactly 18
|
||||
accepted jobs. Existing real-SQLite cancellation/lease/drain tests remain active.
|
||||
|
||||
## 2. SQL offload invalidated fixture assumptions about first poll and publication
|
||||
|
||||
After Host isolation the default-parallel suite finished in 171.57 seconds:
|
||||
954 passed, three failed, one existing ignored. Actual errors were:
|
||||
|
||||
- removal subscriber saw a legitimate queued Runtime stop Upsert before Removed;
|
||||
- external session-open expiry exceeded its one-second bound while its DB lane
|
||||
was still shared across other fixtures;
|
||||
- grant replacement returned success because one initial poll had only submitted
|
||||
the offloaded authority read, not reached the held original-grant lock.
|
||||
|
||||
The ordinary DB fixtures were isolated too. The one-second expiry assertion was
|
||||
not increased. Grant testing now drains its actual authority read then polls the
|
||||
saved response to reach the held lock before replacing the grant. Removal testing
|
||||
observes the committed stopped projection before taking the removal baseline.
|
||||
The Removed payload assertion is unchanged. These are observable synchronization,
|
||||
not sleeps or retries. Focused expiry, grant, and removal tests all passed.
|
||||
|
||||
A captured cleanup job also attempted to poll a Tokio timer after its fixture
|
||||
runtime had shut down. The terminal checker fixture now retains that original
|
||||
runtime through accepted Host and cleanup jobs, then stops projection producers
|
||||
and drains actual DB work before ending. Terminal SQL state is not treated as
|
||||
actual cleanup completion. The subsequent full run contained only the three
|
||||
intentional executor panic diagnostics, not the shutdown-context panic.
|
||||
|
||||
The synchronization/teardown changes are in `df9d056d`.
|
||||
|
||||
## 3. Restore intent's DEFERRED read-to-write upgrade returned SQLite BUSY
|
||||
|
||||
The next default-parallel run finished in 171.48 seconds with 956 passed, two
|
||||
failed, one existing ignored. Both failures were SQLite `DatabaseBusy`, extended
|
||||
code 5, not capacity rejection:
|
||||
|
||||
- production-profile zero-input Orchestrator restore;
|
||||
- pending manual Worker claim restore reconciliation (500 instead of 409).
|
||||
|
||||
Both failed alone too. `target/t748-busy-origin.log` captured the error and a
|
||||
backtrace through `pin_internal_worker_restore_intent` -> the transaction helper
|
||||
-> internal restore -> manual claim. The journal read authority first in a
|
||||
DEFERRED transaction while the offloaded projection's other SQLite connection
|
||||
could write. Such an upgrade can return BUSY without invoking SQLite's busy
|
||||
handler; treating this as a retryable whole mutation would be incorrect.
|
||||
|
||||
A new real-SQLite regression holds an IMMEDIATE writer on another connection,
|
||||
registers an owned per-connection busy observer, and attempts one journal pin.
|
||||
The old helper fails deterministically with BUSY/5 before waiting for ownership
|
||||
(`target/t748-deferred-upgrade-red.log`, 0.23 seconds). The fixed helper waits
|
||||
before its first authority read, while a current-thread timer and control task
|
||||
progress. After releasing the writer it persists exactly one requested tuple;
|
||||
there is no application-level mutation retry.
|
||||
|
||||
`f597f1b5` makes this existing mutation helper explicitly
|
||||
`with_immediate_transaction`. Restore journal and deletion authority reads,
|
||||
mutation and commit remain in one transaction. Focused DB/HTTP tests (8), the
|
||||
original restore and claim cases, and deletion tests (13) passed.
|
||||
|
||||
## 4. Why a pre-dispatch failure could become the 600-second suite timeout
|
||||
|
||||
The original long-running logging case joined an HTTP request with a fake provider
|
||||
waiting for command dispatch. On pre-dispatch rejection the HTTP future can finish
|
||||
with 503 while the receiver remains Pending: its channel is still owned by the
|
||||
fixture, but no command will arrive. An unbounded `join!` waits forever in that
|
||||
state and hides the actual error behind the outer suite timeout.
|
||||
|
||||
The permanent `saturated_host_before_command_start_returns_availability_without_a_provider_operation`
|
||||
regression uses the real signed HTTP route with a controlled full Host budget. It
|
||||
observes the final 503/`db_capacity_exceeded` response and the still-live Pending
|
||||
provider receiver, with no process or command dispatch. The original logging
|
||||
fixture now asserts an early HTTP response before waiting for both branches, so
|
||||
an unexpected rejection reports its body immediately rather than hanging. This
|
||||
reproduces the causal timeout mechanism; the old log itself has no stack trace
|
||||
from the hung provider future.
|
||||
|
||||
The nine logging cases pass together at default parallelism (`49bfbfb2`).
|
||||
|
||||
## Artifact/source isolation during comparison
|
||||
|
||||
Initially the archived baseline used the existing Cargo target directory. That
|
||||
replaced a same-package artifact/fingerprint; one later post-fix representative
|
||||
command ran the baseline's 932-test binary (and `db_executor --list` had no tests).
|
||||
That result was rejected as candidate evidence. All later candidate checks/tests
|
||||
use `--target-dir target/t748-candidate-build`, separate from the baseline cache.
|
||||
This changes artifact location, not test parallelism, features or admission policy.
|
||||
The fresh candidate build listed the expected executor tests and total test count.
|
||||
|
||||
## Validation and remaining composition boundary
|
||||
|
||||
Final code source `49bfbfb2` passed the full default-parallel Server suite,
|
||||
including the command-overload case: 960 passed, zero failures, one existing
|
||||
ignored migration fixture; 11 binary tests, library duration 175.29 seconds.
|
||||
The only captured panics were the three deliberate unknown-outcome tests.
|
||||
Root `cargo check` compile closure passed (including TUI/Server), as did Workdir
|
||||
default (153 tests) and `http-client` (156 tests) full suites, format/diff checks
|
||||
and state-contract names (558 matches, zero unapproved, zero stale exceptions).
|
||||
These commands used the isolated candidate artifact directory; libtest default
|
||||
parallelism and the production budgets were unchanged. The MR records the exact
|
||||
published source, which also includes this report and the boundary document.
|
||||
|
||||
No latency benchmark, production incident measurement, target integration,
|
||||
Ticket conclusion, or running-environment update is claimed. T743 has now
|
||||
published its caller inventory/source; its owner will compose the T748 facade and
|
||||
new callers, then obtain review of that exact combined source. T748 does not copy
|
||||
those domain APIs or wait on the former unpublished branch.
|
||||
@@ -2,6 +2,12 @@
|
||||
|
||||
## Source identity and composition
|
||||
|
||||
This is the retained pre-composition caller inventory published with T-743
|
||||
`9b0d42fdbc2920736c3418bb894f26140e962bbc`. Its descriptions of direct callers
|
||||
below identify the work to move, not the current execution placement. The later
|
||||
normal merge, actual DB/Host placement, ten new regressions, full-suite evidence
|
||||
and unresolved upstream review findings are in `T-743-T748-composition.md`.
|
||||
|
||||
This inventory belongs to the T-743 commit carrying this file on `work/T-743-ticket-public-surfaces` (T-742 base `56795fb92cb9def9bcd701bd51d92e99526f8eff`). Its exact published SHA and MR ID are recorded in the Ticket publication handoff. It is not a copy of the unpublished T-748 adapter and is not evidence that async DB execution has been integrated or reviewed.
|
||||
|
||||
Normal-merge the validated public T-748 source into this existing work branch. Keep the MR's source/target selectors. Recheck the concrete new call graph below against the merged implementation; no placeholder executor, second connection authority, transaction split, fresh integration branch or force push is needed.
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# T-743 / T-748 composition: execution-boundary evidence
|
||||
|
||||
## Source and status
|
||||
|
||||
T-743 continues on `work/T-743-ticket-public-surfaces`, MR
|
||||
`01a12d19-b61b-79a2-baca-89241fd3c3d3` → `develop`. The published T-748
|
||||
source `66bb9d0b825d72e20923003b273b787007f3aed8` was normally fetched and
|
||||
merged into that same source branch, without conflicts, replacement selectors,
|
||||
force push, or a copied execution facade. The composition commit carrying this
|
||||
report also connects the added callers below. Its exact published hash is in the
|
||||
Ticket handoff and provider-resolved MR source.
|
||||
|
||||
**This is a validated intermediate composition, not final independent approval.**
|
||||
T-748 MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq3 requests changes on this
|
||||
upstream source. Its two major findings are not fixed by the new assignment
|
||||
caller work: `T748-DB-NETWORK-LANE` (Subjektiv mixed Runtime/DB placement) and
|
||||
`T748-ASYNC-PROTOCOL-SIGNING` (cold protocol observation source signing).
|
||||
That authoritative verdict was checked again after the combined full suite;
|
||||
no newer published upstream source or verdict was present. The T-748 owner was
|
||||
informed through its Ticket of the composition and requested to provide the
|
||||
normally advanced validated fix source. Do not interpret a green suite as proof
|
||||
that those untested boundaries are safe. Normal-merge that fix into this same
|
||||
T-743 source, revalidate, and independently review the exact combined source.
|
||||
|
||||
## Caller placement
|
||||
|
||||
- Canonical and Browser assignment pages and sanitized diagnostics share one
|
||||
implementation. Workspace scope, Ticket reference resolution and each whole
|
||||
synchronous Store read run on the shared bounded DB owner. None is left as a
|
||||
synchronous pre-call on a Tokio request thread.
|
||||
- Typed Ticket query, including principal/handling/current-closure filtering
|
||||
before pagination, retains T-748's whole authority/read/enrichment DB job.
|
||||
The regression holds the **authority's actual candidate connection**, not the
|
||||
separate API Store connection used by only some enrichment paths. This also
|
||||
proves the empty-result path and wrong-principal filters offload correctly.
|
||||
- Account assignment answers derive attribution from Backend-authenticated
|
||||
context inside the owned DB closure. Runtime answers preserve the live current
|
||||
Orchestrator discovery check on the finite Host lane, because Runtime worker
|
||||
lookup can perform remote HTTP. The DB closure derives Runtime attribution
|
||||
without I/O and the unchanged answer helper validates current singleton and
|
||||
live binding **again under its IMMEDIATE write transaction**, including replay.
|
||||
Target principal and request-body prose never supply author authority.
|
||||
- The same answer helper keeps source authority, actionable-assignment gate,
|
||||
assignment/event CAS, full operation fingerprint/replay, body, audit, receipt
|
||||
and commit together. There is no await within that transaction and no automatic
|
||||
mutation retry.
|
||||
- The accepted SQL job admits the existing finite Host attention continuation
|
||||
**after commit and before returning its result**. It inherits the request
|
||||
lifecycle source, owns lookup/Runtime dispatch independently of the waiter and
|
||||
does not wait on Runtime/network on the sole DB worker. A live HTTP waiter
|
||||
observes best-effort notification completion; losing the waiter does not
|
||||
cancel accepted SQL or accepted publication. Rejected Host publication retains
|
||||
the existing saved-input scan/recovery and diagnostic behavior, not rollback
|
||||
or a synthetic successful delivery.
|
||||
- Normal Tools/WIP still converge on the canonical HTTP contracts. No second
|
||||
ledger, query adapter, source authority, read pool or delivery implementation
|
||||
was added. Existing T-748 set/clear/start/delivery boundaries remain intact.
|
||||
|
||||
## New regression proof
|
||||
|
||||
`crates/workspace-server/src/server_assignment_db_execution_tests.rs` contains
|
||||
10 tests on fixture-owned bounded DB/Host owners, temporary databases and explicit
|
||||
observable gates. Parent inspected the child's committed tool output, the actual
|
||||
new test code and then the combined default-parallel suite, rather than treating
|
||||
an end notification as proof. The child's delegated file scope was released.
|
||||
|
||||
The regressions prove:
|
||||
|
||||
1. Assignment/diagnostic read and Ticket reference resolution wait off the
|
||||
current-thread executor on their actual Store connection mutex.
|
||||
2. Typed principal/handling/closure query matrices, including empty results,
|
||||
hold the candidate connection and filter before the page limit.
|
||||
3. A real second SQLite connection's IMMEDIATE writer reaches the answer's owned
|
||||
busy observer while the same Tokio runtime's timer and independent control
|
||||
progress; one transaction preserves the answer/event/audit/receipt.
|
||||
4. Saturated DB admission returns HTTP 503 and does not accept an answer.
|
||||
5. Cancelling an accepted answer waiter retains its actual job slot/lifecycle
|
||||
through SQL and Host publication; body/audit/receipt persist, dispatch is once,
|
||||
and actual cleanup releases all observed leases.
|
||||
6. Successful HTTP answer waits for its admitted publication without occupying
|
||||
the DB worker; an independent DB callback can progress during publication.
|
||||
7. Account and Browser receipt replay, stale CAS, operation mismatch and bad
|
||||
cursor/error mapping retain their domain meanings.
|
||||
8. Real signed non-Orchestrator source and signed proof replay are denied with
|
||||
no answer/receipt; signed current Orchestrator canonical/Browser replay shares
|
||||
one source-bound receipt, and another source cannot replay it.
|
||||
9. Saturated **2-worker/16-queued Host** live-discovery rejects canonical/Browser
|
||||
Runtime answers before SQL admission, without receipt/audit/publication or
|
||||
retained lifecycle, while independent DB callbacks progress. This last test
|
||||
injects authenticated context at generated handlers to isolate discovery
|
||||
admission; the preceding tests exercise actual signed-proof middleware.
|
||||
|
||||
Read/query contention intentionally uses the actual connection mutex; answer
|
||||
contention uses a separate real SQLite writer. Neither is described as a full
|
||||
remote Runtime latency benchmark. No fixed completion sleeps, mutation retry,
|
||||
parallelism reduction, process-global fixture budgets or new ignored tests were
|
||||
introduced. Fixtures drain accepted Host/cleanup jobs, shut down projection
|
||||
producers, then drain DB jobs before ending their original Tokio driver. Owned
|
||||
busy-handler userdata and SQLite locks unregister/release through RAII.
|
||||
|
||||
## Validation on this composition
|
||||
|
||||
- Root `cargo check`: passed, normal default-member compile closure.
|
||||
- `cargo test -p yoi-workspace-server --lib assignment_db_execution_tests`:
|
||||
**10 passed, 0 failed**, 2.63s (child committed output). Parent reran the first
|
||||
nine after separating Runtime discovery and included all ten in the full suite.
|
||||
- `cargo test -p yoi-workspace-server`: original default parallelism,
|
||||
**985 library passed, 0 failed, 1 pre-existing ignored**, 176.60s;
|
||||
**11 binary passed**, doc tests passed. Log:
|
||||
`target/t743-combined-server-suite.log`.
|
||||
- `cargo test -p workdir`: **153 passed** across library/integration targets;
|
||||
`cargo test -p workdir --features http-client`: **156 passed**, doc tests passed.
|
||||
Logs: `target/t743-combined-workdir.log`, `target/t743-combined-workdir-http.log`.
|
||||
- Combined Web check: **0 errors, 0 warnings**; static production build passed
|
||||
with the existing chunk-size advisory. Web tests **613 passed**, components
|
||||
**318 passed**, Chromium Ticket/principal browser tests **7 passed**.
|
||||
Logs: `target/t743-combined-{web-check,web-build,web-tests,components,browser}.log`.
|
||||
The last change is Server execution placement/test code, not a UI change.
|
||||
- All six server-api TypeScript generator `--check` commands passed: ticket,
|
||||
repository OpenAPI, runtime, workdir, Worker launch and companion.
|
||||
- `cargo fmt --all -- --check`, `git diff --check HEAD` and
|
||||
`env -u LD_LIBRARY_PATH deno run --allow-read --allow-run=git tools/check-state-contract-names.ts`
|
||||
passed; naming: **558 matching lines, 0 unapproved, 0 stale exceptions**.
|
||||
|
||||
Earlier T-743 public-surface, five-crate, Prompt/Tool, visual and real-Worker
|
||||
fixture evidence is retained in `T-743-public-surface-validation.md`. Scripted
|
||||
model/loopback browser fixtures do not attest real model judgment or all T-744
|
||||
live paths. Target integration, Ticket/MR conclusion, live update and retained
|
||||
Worker/Workdir cleanup were not performed. Fresh independent combined-source
|
||||
review remains pending the upstream fixes, not silently waived.
|
||||
@@ -26,7 +26,7 @@ Canonical `/workspaces/{workspace_id}/tickets/{ticket_id}` and `/api/w/{workspac
|
||||
|
||||
## Validation
|
||||
|
||||
Passed on the combined working snapshot:
|
||||
Passed on the initial T-743 public-surface snapshot (before T-748 composition); later combined execution-boundary evidence is in `T-743-T748-composition.md`:
|
||||
|
||||
- Root `cargo check` (normal TUI/Server compile closure).
|
||||
- `cargo test --no-fail-fast -p ticket -p worker-runtime -p worker -p yoi-workspace-server -p server-api`: all selected crate targets passed. Library counts: server-api 98; ticket 99; worker 994; worker-runtime 417; workspace-server 946 with one existing ignored test. Included integration/binary targets also passed. Log: `target/t743-changed-crates-final.log`.
|
||||
@@ -51,7 +51,7 @@ List/Orchestrator final evidence: `t743-list-final-reviewed-light` and `t743-lis
|
||||
|
||||
## Explicit remaining work and limits
|
||||
|
||||
T-748's DB execution facade is **not** copied or integrated. Its latest notified candidate is local-only and its full Server suite has unresolved failures. When T-748 publishes a validated source, normal-merge it, place each new whole synchronous domain helper/authority query into the common bounded DB execution boundary, keep the answer transaction intact, and leave Runtime/network waiting off the DB worker. Recheck cancellation/overload and commit-before-publication at the combined boundary. The exact combined published source then needs independent trusted review.
|
||||
The published T-748 source `66bb9d0b825d72e20923003b273b787007f3aed8` has now been normally merged into the same T-743 source and the added callers connected to its existing bounded owner. The answer transaction remains intact; Runtime live discovery and post-commit dispatch use the finite Host lane. Combined full Server/Workdir and ten new boundary regressions pass; see `T-743-T748-composition.md` for exact proof and distinctions. However T-748 MR seq3 has two unresolved major findings on other Subjektiv/network and cold protocol signing paths. A validated upstream fix source, normal composition/revalidation and fresh independent exact combined-source review remain necessary. This is not final approval.
|
||||
|
||||
Current evidence does not attest actual model judgment quality. Prompt/Tool fixtures are scripted inclusion/schema/transport tests; Chromium uses a loopback API fixture. The real-Worker Server fixture uses a scripted model with actual Controller/session/Runtime acceptance and canonical processing synchronization. None is a live migration/rollout, proof of actual processing for every production delivery, or completion of T-744's full real-path validation.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user