fix: compose bounded execution with Ticket public callers
This commit is contained in:
+145
-16
@@ -432,16 +432,23 @@ mod client {
|
||||
use super::*;
|
||||
use crate::{Workdir, WorkdirSession};
|
||||
|
||||
/// Provides a fresh bearer token for each Runtime request. Backend
|
||||
/// implementations can mint short-lived capability tokens without making a
|
||||
/// Worker-bound session expire with the token used to open it.
|
||||
/// An owned, Send authorization future that can offload synchronous authority
|
||||
/// work without borrowing request data or the authorization provider.
|
||||
pub type WorkdirHttpAuthorizationFuture = std::pin::Pin<
|
||||
Box<dyn std::future::Future<Output = Result<String, WorkdirError>> + Send + 'static>,
|
||||
>;
|
||||
|
||||
/// Provides a fresh bearer token asynchronously for each Runtime request.
|
||||
/// Backend implementations can mint short-lived capability tokens without
|
||||
/// making a Worker-bound session expire with the token used to open it.
|
||||
/// Blocking DB/signing work must be offloaded by the implementation.
|
||||
pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: &str,
|
||||
path_and_query: &str,
|
||||
body: &[u8],
|
||||
) -> Result<String, WorkdirError>;
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture;
|
||||
}
|
||||
|
||||
struct FixedBearerToken(Arc<str>);
|
||||
@@ -455,11 +462,12 @@ mod client {
|
||||
impl WorkdirHttpAuthorization for FixedBearerToken {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
_method: &str,
|
||||
_path_and_query: &str,
|
||||
_body: &[u8],
|
||||
) -> Result<String, WorkdirError> {
|
||||
Ok(self.0.to_string())
|
||||
_method: String,
|
||||
_path_and_query: String,
|
||||
_body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let token = self.0.clone();
|
||||
Box::pin(async move { Ok(token.to_string()) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,7 +517,9 @@ mod client {
|
||||
)?;
|
||||
let body = serde_json::to_vec(&request)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
|
||||
let token = authorization.bearer_token("POST", url.path(), &body)?;
|
||||
let token = authorization
|
||||
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
|
||||
.await?;
|
||||
let response = client
|
||||
.post(url)
|
||||
.bearer_auth(token)
|
||||
@@ -560,7 +570,10 @@ mod client {
|
||||
let operation = WorkdirSessionOperationRequest { operation };
|
||||
let body = serde_json::to_vec(&operation)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
|
||||
let token = self.authorization.bearer_token("POST", url.path(), &body)?;
|
||||
let token = self
|
||||
.authorization
|
||||
.bearer_token("POST".to_string(), url.path().to_string(), body.clone())
|
||||
.await?;
|
||||
let response = self
|
||||
.client
|
||||
.post(url)
|
||||
@@ -864,7 +877,10 @@ mod client {
|
||||
&self.base_url,
|
||||
&["v1", "workdir-sessions", self.session_id.as_str()],
|
||||
)?;
|
||||
let token = self.authorization.bearer_token("DELETE", url.path(), &[])?;
|
||||
let token = self
|
||||
.authorization
|
||||
.bearer_token("DELETE".to_string(), url.path().to_string(), Vec::new())
|
||||
.await?;
|
||||
let response = self
|
||||
.client
|
||||
.delete(url)
|
||||
@@ -918,11 +934,124 @@ mod client {
|
||||
WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod authorization_tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct RejectAuthorization {
|
||||
requests: Arc<Mutex<Vec<(String, String, Vec<u8>)>>>,
|
||||
}
|
||||
|
||||
impl WorkdirHttpAuthorization for RejectAuthorization {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let requests = self.requests.clone();
|
||||
Box::pin(async move {
|
||||
tokio::task::yield_now().await;
|
||||
let mut requests = requests.lock().unwrap();
|
||||
requests.push((method, path_and_query, body));
|
||||
Err(WorkdirError::Unavailable(format!(
|
||||
"authorization rejected request {}",
|
||||
requests.len()
|
||||
)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_operate_close_await_fresh_authorization_and_do_not_send_when_rejected() {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let base_url: Url = format!("http://{}", listener.local_addr().unwrap())
|
||||
.parse()
|
||||
.unwrap();
|
||||
let client = Client::builder().no_proxy().build().unwrap();
|
||||
let authorization = Arc::new(RejectAuthorization::default());
|
||||
let workdir = Workdir::new("w");
|
||||
let open_request = OpenWorkdirSessionRequest {
|
||||
owner_worker_id: Some("worker".to_string()),
|
||||
};
|
||||
let error = RemoteWorkdirSession::open_with_authorization(
|
||||
client.clone(),
|
||||
base_url.clone(),
|
||||
authorization.clone(),
|
||||
workdir.id().clone(),
|
||||
open_request.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 1"
|
||||
));
|
||||
|
||||
// Seed an already-open session to exercise the public operation and
|
||||
// close boundaries independently of the rejected open request.
|
||||
let remote = RemoteWorkdirSession {
|
||||
client,
|
||||
base_url,
|
||||
authorization: authorization.clone(),
|
||||
workdir,
|
||||
session_id: WorkdirSessionId::new("s").unwrap(),
|
||||
capabilities: WorkdirSessionCapabilities::EMPTY,
|
||||
closed: AtomicBool::new(false),
|
||||
};
|
||||
let stat = StatRequest {
|
||||
path: crate::WorkdirPath::new("file").unwrap(),
|
||||
};
|
||||
let error = remote.stat(stat.clone()).await.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 2"
|
||||
));
|
||||
let error = remote.close().await.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == "authorization rejected request 3"
|
||||
));
|
||||
assert_eq!(
|
||||
*authorization.requests.lock().unwrap(),
|
||||
vec![
|
||||
(
|
||||
"POST".to_string(),
|
||||
"/v1/working-directories/w/sessions".to_string(),
|
||||
serde_json::to_vec(&open_request).unwrap(),
|
||||
),
|
||||
(
|
||||
"POST".to_string(),
|
||||
"/v1/workdir-sessions/s/operations".to_string(),
|
||||
serde_json::to_vec(&WorkdirSessionOperationRequest {
|
||||
operation: WorkdirSessionOperation::Stat(stat),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
(
|
||||
"DELETE".to_string(),
|
||||
"/v1/workdir-sessions/s".to_string(),
|
||||
Vec::new(),
|
||||
),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
listener.accept().unwrap_err().kind(),
|
||||
std::io::ErrorKind::WouldBlock
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub use self::RemoteWorkdirSession as ClientSession;
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-client")]
|
||||
pub use client::{ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization};
|
||||
pub use client::{
|
||||
ClientSession as RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
@@ -147,7 +147,7 @@ async fn assignment_delivery_response_loss_returns_immutable_payload_after_progr
|
||||
.update_ticket_assignment_tracking("w", "tracking", &update, &source)
|
||||
.unwrap();
|
||||
drop(store);
|
||||
let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.list_pending_ticket_assignment_deliveries("w", 100)
|
||||
@@ -208,7 +208,7 @@ async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt()
|
||||
let path = directory.path().join("workspace.db");
|
||||
let input = pending.input_event_id.clone();
|
||||
std::thread::spawn(move || {
|
||||
let store = SqliteWorkspaceStore::open(path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
|
||||
barrier.wait();
|
||||
store
|
||||
.reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source())
|
||||
@@ -568,7 +568,7 @@ async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_co
|
||||
"original rendered prompt"
|
||||
);
|
||||
drop(store);
|
||||
let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.prepare_ticket_assignment_delivery_content(
|
||||
@@ -634,7 +634,7 @@ fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_l
|
||||
let operations = stored_fixture_rows(&conn, "ticket_assignment_operations");
|
||||
let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history");
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let pending = store
|
||||
.list_pending_ticket_assignment_deliveries("w", 100)
|
||||
.unwrap();
|
||||
@@ -684,7 +684,7 @@ async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once
|
||||
let barrier = barrier.clone();
|
||||
let path = directory.path().join("workspace.db");
|
||||
std::thread::spawn(move || {
|
||||
let store = SqliteWorkspaceStore::open(path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
|
||||
barrier.wait();
|
||||
delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source())
|
||||
.unwrap()
|
||||
@@ -716,8 +716,8 @@ fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() {
|
||||
let conn = tracking_schema91_fixture(&path);
|
||||
migrate_ticket_tracking_v91_to_v92(&conn).unwrap();
|
||||
drop(conn);
|
||||
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
let objects = |store: &SqliteWorkspaceStore| {
|
||||
store.with_conn(|conn| {
|
||||
let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?;
|
||||
@@ -741,7 +741,7 @@ async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() {
|
||||
.unwrap();
|
||||
drop(store);
|
||||
assert!(
|
||||
matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update"))
|
||||
matches!(SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update"))
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ async fn tracking_store_every_replay_fences_current_singleton_and_authenticated_
|
||||
);
|
||||
// Model the authority change after authenticated source capture, before
|
||||
// BEGIN IMMEDIATE on a second connection. Receipt presence cannot bypass it.
|
||||
let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
second.with_conn_mut(|conn| {
|
||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
match change {
|
||||
@@ -233,7 +233,7 @@ async fn tracking_store_same_principal_reissue_and_exact_retries_keep_original_r
|
||||
.is_err()
|
||||
);
|
||||
drop(store);
|
||||
let reopened = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.get_ticket_assignment_tracking("w", "tracking", "a")
|
||||
@@ -441,7 +441,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
.collect::<Vec<_>>(),
|
||||
vec![92, 93]
|
||||
);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, 93);
|
||||
@@ -525,7 +525,7 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
.list_ticket_assignment_tracking_history("w", "pending", "a-pending")
|
||||
.unwrap();
|
||||
drop(store);
|
||||
let reopened = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.list_ticket_assignment_tracking_history("w", "pending", "a-pending")
|
||||
@@ -550,7 +550,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
|
||||
let releases_before = stored_fixture_rows(&conn, "ticket_assignment_work_releases");
|
||||
drop(conn);
|
||||
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let backend = ticket::SqliteTicketBackend::open_verified(&path, "w").unwrap();
|
||||
let reopen = || {
|
||||
let observed = backend.show("historical-done".into()).unwrap();
|
||||
@@ -645,7 +645,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
|
||||
);
|
||||
// A second observer holds the same Assign baseline. Only the first
|
||||
// explicit writer may consume it; BEGIN IMMEDIATE serializes the CAS.
|
||||
let second = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let second = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
second
|
||||
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
|
||||
@@ -740,7 +740,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first
|
||||
}).unwrap();
|
||||
drop(second);
|
||||
drop(store);
|
||||
let reopened_store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let reopened_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
reopened_store
|
||||
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
|
||||
@@ -796,7 +796,7 @@ fn tracking_store_schema91_missing_assign_audit_remains_unknown_without_cas_base
|
||||
)
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let unknown = store
|
||||
.get_ticket_assignment_tracking("w", "historical-done", "a-historical-done")
|
||||
.unwrap()
|
||||
@@ -1068,7 +1068,7 @@ async fn orchestrator_assignment_all_runtime_retries_fence_singleton_and_binding
|
||||
&source,
|
||||
)
|
||||
.unwrap();
|
||||
let second = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap();
|
||||
let second = SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")).unwrap();
|
||||
second.with_conn_mut(|conn| {let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;match change {
|
||||
"replace" => {tx.execute("UPDATE worker_singleton_owners SET worker_id='replacement',generation=2 WHERE workspace_id='w'",[])?;},
|
||||
"revoke" => {tx.execute("UPDATE workspace_runtime_bindings SET revoked_at='revoked' WHERE workspace_id='w'",[])?;},
|
||||
|
||||
@@ -89,12 +89,30 @@ pub fn parse_cookie(headers: &HeaderMap, cookie_name: &str) -> Option<String> {
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
|
||||
store: &S,
|
||||
pub async fn resolve_request_actor(
|
||||
store: std::sync::Arc<dyn ControlPlaneStore>,
|
||||
headers: &HeaderMap,
|
||||
cookie_name: &str,
|
||||
) -> Result<Option<RequestActor>> {
|
||||
if let Some(token) = parse_bearer(headers) {
|
||||
let bearer = parse_bearer(headers);
|
||||
let session = parse_cookie(headers, cookie_name);
|
||||
if bearer.is_none() && session.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let executor = store.db_executor();
|
||||
executor
|
||||
.run(move || resolve_request_actor_sync(store.as_ref(), bearer, session))
|
||||
.await
|
||||
}
|
||||
|
||||
// One owned synchronous stage includes token usage marking and the user read.
|
||||
// Do not call async store wrappers here: they use the same executor themselves.
|
||||
fn resolve_request_actor_sync(
|
||||
store: &dyn ControlPlaneStore,
|
||||
bearer: Option<String>,
|
||||
session: Option<String>,
|
||||
) -> Result<Option<RequestActor>> {
|
||||
if let Some(token) = bearer {
|
||||
let token_hash = token_hash(&token);
|
||||
if let Some(api_token) = store.resolve_api_token(&token_hash)? {
|
||||
if api_token
|
||||
@@ -109,7 +127,7 @@ pub async fn resolve_request_actor<S: ControlPlaneStore + ?Sized>(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(session_token) = parse_cookie(headers, cookie_name) {
|
||||
if let Some(session_token) = session {
|
||||
if let Some(session) = store.resolve_browser_session(&token_hash(&session_token))? {
|
||||
if is_expired(&session.expires_at) {
|
||||
return Ok(None);
|
||||
@@ -173,6 +191,182 @@ pub fn auth_error(code: &str, message: &str) -> Error {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::{
|
||||
AccountRecord, ApiTokenRecord, BrowserSessionRecord, SqliteWorkspaceStore, UserRecord,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn seeded_store() -> Arc<SqliteWorkspaceStore> {
|
||||
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::in_memory_for_test()
|
||||
.unwrap()
|
||||
.with_db_executor(executor),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-auth".into(),
|
||||
kind: "user".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_user(&UserRecord {
|
||||
user_id: "user-auth".into(),
|
||||
account_id: "account-auth".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.create_browser_session(&BrowserSessionRecord {
|
||||
session_id: "session-auth".into(),
|
||||
token_hash: token_hash("session-secret"),
|
||||
user_id: "user-auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: "9999-01-01T00:00:00Z".into(),
|
||||
revoked_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_token_auth_marks_usage_and_takes_precedence_over_cookie() {
|
||||
let store = seeded_store();
|
||||
let hash = token_hash("api-secret");
|
||||
store
|
||||
.create_api_token(&ApiTokenRecord {
|
||||
token_id: "token-auth".into(),
|
||||
token_hash: hash.clone(),
|
||||
user_id: "user-auth".into(),
|
||||
label: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: None,
|
||||
revoked_at: None,
|
||||
last_used_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::AUTHORIZATION, "Bearer api-secret".parse().unwrap());
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let actor = resolve_request_actor(store.clone(), &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(actor.user_id, "user-auth");
|
||||
assert_eq!(actor.auth_method, ActorAuthMethod::ApiToken);
|
||||
assert!(
|
||||
store
|
||||
.resolve_api_token(&hash)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.last_used_at
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_api_token_does_not_fall_back_to_valid_browser_session() {
|
||||
let store = seeded_store();
|
||||
let hash = token_hash("expired-secret");
|
||||
store
|
||||
.create_api_token(&ApiTokenRecord {
|
||||
token_id: "token-auth".into(),
|
||||
token_hash: hash.clone(),
|
||||
user_id: "user-auth".into(),
|
||||
label: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
expires_at: Some("2000-01-01T00:00:00Z".into()),
|
||||
revoked_at: None,
|
||||
last_used_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::AUTHORIZATION,
|
||||
"Bearer expired-secret".parse().unwrap(),
|
||||
);
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
assert!(
|
||||
resolve_request_actor(store.clone(), &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.resolve_api_token(&hash)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.last_used_at
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_bearer_can_fall_back_to_valid_browser_session() {
|
||||
let store = seeded_store();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::AUTHORIZATION,
|
||||
"Bearer unknown-secret".parse().unwrap(),
|
||||
);
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let actor = resolve_request_actor(store, &headers, "session")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(actor.user_id, "user-auth");
|
||||
assert_eq!(actor.auth_method, ActorAuthMethod::BrowserSession);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_actor_lookup_returns_capacity_error_not_anonymous_actor() {
|
||||
let store = seeded_store();
|
||||
let executor = store.db_executor();
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker_executor = executor.clone();
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_executor
|
||||
.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
release_rx.recv().unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut queued = Box::pin(executor.run(|| -> Result<()> { Ok(()) }));
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::COOKIE, "session=session-secret".parse().unwrap());
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(1),
|
||||
resolve_request_actor(store, &headers, "session"),
|
||||
)
|
||||
.await;
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap().unwrap();
|
||||
queued.await.unwrap();
|
||||
assert!(matches!(
|
||||
result.expect("capacity rejection must not wait for queued SQL"),
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_code_uses_the_uuid_random_tail() {
|
||||
|
||||
@@ -3349,7 +3349,7 @@ mod tests {
|
||||
async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "w".into(),
|
||||
@@ -3424,7 +3424,7 @@ mod tests {
|
||||
async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "w".into(),
|
||||
@@ -3557,7 +3557,7 @@ mod tests {
|
||||
write_ticket(dir.path(), "00000000001J5", "Second ticket", "queued");
|
||||
write_ticket(dir.path(), "00000000001J6", "Third ticket", "planning");
|
||||
let db_path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-test".to_string(),
|
||||
@@ -3989,7 +3989,7 @@ VALUES ('workspace-test', 'ticket', 4);
|
||||
async fn objective_mutations_write_sqlite_records_and_audit_events() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db_path = dir.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-test".to_string(),
|
||||
|
||||
@@ -795,7 +795,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn open_store() -> SqliteWorkspaceStore {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
store
|
||||
}
|
||||
@@ -1454,7 +1454,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_candidate_is_never_persisted() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let main = current.snapshot.get(&path(MAIN_CONFIG_ENTRYPOINT)).unwrap();
|
||||
@@ -1478,7 +1478,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn valid_candidate_commits_snapshot_content_and_provenance_atomically() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let committed = store
|
||||
@@ -1496,7 +1496,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_cas_cannot_overwrite_newer_tree() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let request = commit_request(¤t, vec![update_main(¤t, "{ answer = 42; }")]);
|
||||
@@ -1514,7 +1514,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_content_remains_retrievable_after_later_commit() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store.upsert_workspace(&workspace()).await.unwrap();
|
||||
let current = store.load_workspace_config("w-config").unwrap().unwrap();
|
||||
let first = store
|
||||
@@ -1772,7 +1772,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn migration_creates_config_authority_tables() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
for table in [
|
||||
|
||||
@@ -0,0 +1,957 @@
|
||||
//! Bounded execution of whole synchronous DB operations, independent of Tokio.
|
||||
//!
|
||||
//! Accepted jobs belong to the workers, not their async waiters. Cancelling a
|
||||
//! waiter does not cancel or retry SQL, release capacity, or release its lease.
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
future::Future,
|
||||
num::NonZeroUsize,
|
||||
panic::{AssertUnwindSafe, catch_unwind},
|
||||
sync::{Arc, Mutex, OnceLock, mpsc},
|
||||
};
|
||||
use tokio::sync::{Notify, oneshot};
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// The default is one FIFO worker and at most 32 waiting operations.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct DbExecutorLimits {
|
||||
pub workers: NonZeroUsize,
|
||||
pub queued: NonZeroUsize,
|
||||
}
|
||||
|
||||
impl Default for DbExecutorLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(32).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A request can retain its lifecycle authority until actual SQL finishes.
|
||||
/// Retention happens at submission, while the caller's task-local scope exists.
|
||||
pub(crate) trait DbOperationLifetime: Send + Sync {
|
||||
fn retain(&self) -> Box<dyn Send>;
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static OPERATION_LIFETIME: Arc<dyn DbOperationLifetime>;
|
||||
}
|
||||
|
||||
pub(crate) async fn with_operation_lifetime<F: Future>(
|
||||
lifetime: Arc<dyn DbOperationLifetime>,
|
||||
future: F,
|
||||
) -> F::Output {
|
||||
OPERATION_LIFETIME.scope(lifetime, future).await
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Accepted {
|
||||
last_job_id: u64,
|
||||
unfinished: BTreeSet<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Tracking {
|
||||
accepted: Mutex<Accepted>,
|
||||
finished: Notify,
|
||||
}
|
||||
|
||||
struct Completion {
|
||||
tracking: Arc<Tracking>,
|
||||
id: u64,
|
||||
}
|
||||
|
||||
impl Drop for Completion {
|
||||
fn drop(&mut self) {
|
||||
self.tracking
|
||||
.accepted
|
||||
.lock()
|
||||
.unwrap()
|
||||
.unfinished
|
||||
.remove(&self.id);
|
||||
self.tracking.finished.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
struct Job {
|
||||
operation: Box<dyn FnOnce() + Send>,
|
||||
// Field drop order releases authority before completion becomes observable.
|
||||
_lifetime: Option<Box<dyn Send>>,
|
||||
lifetime_source: Option<Arc<dyn DbOperationLifetime>>,
|
||||
_completion: Completion,
|
||||
runtime: Option<tokio::runtime::Handle>,
|
||||
diagnostics: tracing::Dispatch,
|
||||
span: tracing::Span,
|
||||
}
|
||||
|
||||
fn report_unknown(accepted_job_id: u64, phase: &'static str) {
|
||||
tracing::error!(
|
||||
code = "db_operation_outcome_unknown",
|
||||
accepted_job_id,
|
||||
phase,
|
||||
error = %Error::DbOperationOutcomeUnknown,
|
||||
"DB job panicked; outcome unknown, operation will not be retried"
|
||||
);
|
||||
}
|
||||
|
||||
/// Clones share the queue and actual-job budget. Workers retain only the
|
||||
/// receiver, so dropping the last sender closes the queue; accepted jobs finish
|
||||
/// before the dedicated threads exit. No thread is created per store open.
|
||||
#[derive(Clone)]
|
||||
pub struct DbExecutor {
|
||||
sender: Option<mpsc::SyncSender<Job>>,
|
||||
tracking: Arc<Tracking>,
|
||||
max_jobs: usize,
|
||||
}
|
||||
|
||||
impl DbExecutor {
|
||||
pub fn new(limits: DbExecutorLimits) -> Result<Self> {
|
||||
let max_jobs = limits
|
||||
.workers
|
||||
.get()
|
||||
.checked_add(limits.queued.get())
|
||||
.ok_or_else(|| Error::InvalidInput("DB executor limits overflow".into()))?;
|
||||
let (sender, receiver) = mpsc::sync_channel::<Job>(limits.queued.get());
|
||||
let receiver = Arc::new(Mutex::new(receiver));
|
||||
for index in 0..limits.workers.get() {
|
||||
let receiver = receiver.clone();
|
||||
std::thread::Builder::new()
|
||||
.name(format!("workspace-db-{index}"))
|
||||
.spawn(move || {
|
||||
loop {
|
||||
let next = receiver.lock().unwrap().recv();
|
||||
let Ok(job) = next else { break };
|
||||
let Job {
|
||||
operation,
|
||||
_lifetime: lifetime,
|
||||
lifetime_source,
|
||||
_completion: completion,
|
||||
runtime,
|
||||
diagnostics,
|
||||
span,
|
||||
} = job;
|
||||
// Keep diagnostics with their submitting subscriber as
|
||||
// well as the runtime; no process-global override.
|
||||
let _diagnostics = tracing::dispatcher::set_default(&diagnostics);
|
||||
let _span = span.enter();
|
||||
// Preserve the submitting runtime for synchronous Host
|
||||
// post-commit hooks, including reply/lease destructors.
|
||||
// enter() supplies context only: never drive a future
|
||||
// or await/nest an executor submission in a transaction.
|
||||
let _entered = runtime.as_ref().map(tokio::runtime::Handle::enter);
|
||||
// A cancelled waiter's discarded result may itself have
|
||||
// a destructor. Keep both the lease and actual-job slot
|
||||
// until that cleanup finishes, even if it panics.
|
||||
let execute = || {
|
||||
if catch_unwind(AssertUnwindSafe(operation)).is_err() {
|
||||
report_unknown(completion.id, "reply_cleanup");
|
||||
}
|
||||
if catch_unwind(AssertUnwindSafe(|| drop(lifetime))).is_err() {
|
||||
report_unknown(completion.id, "lifetime_cleanup");
|
||||
}
|
||||
};
|
||||
// Synchronous post-commit continuations can admit another
|
||||
// bounded job. Transfer the original lifecycle source as
|
||||
// well, so its authority survives the parent job/waiter.
|
||||
// This does not inject context into detached Tokio tasks.
|
||||
if let Some(source) = lifetime_source {
|
||||
OPERATION_LIFETIME.sync_scope(source, execute);
|
||||
} else {
|
||||
execute();
|
||||
}
|
||||
drop(completion);
|
||||
}
|
||||
})
|
||||
.map_err(|_| Error::DbExecutorUnavailable)?;
|
||||
}
|
||||
Ok(Self {
|
||||
sender: Some(sender),
|
||||
tracking: Arc::default(),
|
||||
max_jobs,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn shared() -> Self {
|
||||
static SHARED: OnceLock<DbExecutor> = OnceLock::new();
|
||||
SHARED
|
||||
.get_or_init(|| {
|
||||
Self::new(DbExecutorLimits::default()).unwrap_or_else(|_| Self {
|
||||
sender: None,
|
||||
tracking: Arc::default(),
|
||||
max_jobs: 0,
|
||||
})
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Submit without waiting for queue capacity. The closure must contain the
|
||||
/// entire synchronous operation/transaction, with no nested async runtime.
|
||||
/// The submitting Tokio handle, when present, is entered for synchronous
|
||||
/// post-commit hooks and worker cleanup; it is never used to block_on SQL.
|
||||
pub async fn run<T, E>(
|
||||
&self,
|
||||
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
|
||||
) -> std::result::Result<T, E>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: From<Error> + Send + 'static,
|
||||
{
|
||||
self.submit(operation)?
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::DbOperationOutcomeUnknown.into()))
|
||||
}
|
||||
|
||||
/// Admit a fire-and-forget operation immediately, without spawning a Tokio
|
||||
/// task or waiting for capacity. Err means the operation was not accepted;
|
||||
/// Ok means the worker owns it through execution and cleanup, even after
|
||||
/// the submitting task is cancelled. A panic emits a structured unknown-
|
||||
/// outcome diagnostic, never an automatic retry. drain() tracks these jobs.
|
||||
pub fn dispatch(&self, operation: impl FnOnce() + Send + 'static) -> Result<()> {
|
||||
self.submit(move || {
|
||||
operation();
|
||||
Ok::<_, Error>(())
|
||||
})
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
/// Shared synchronous admission: neither dropped receivers nor cancelled
|
||||
/// callers remove accepted work from the queue or the actual-job budget.
|
||||
fn submit<T, E>(
|
||||
&self,
|
||||
operation: impl FnOnce() -> std::result::Result<T, E> + Send + 'static,
|
||||
) -> std::result::Result<oneshot::Receiver<std::result::Result<T, E>>, E>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: From<Error> + Send + 'static,
|
||||
{
|
||||
let sender = self
|
||||
.sender
|
||||
.as_ref()
|
||||
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
|
||||
let runtime = tokio::runtime::Handle::try_current().ok();
|
||||
let diagnostics = tracing::dispatcher::get_default(Clone::clone);
|
||||
let span = tracing::Span::current();
|
||||
let lifetime_source = OPERATION_LIFETIME.try_with(Clone::clone).ok();
|
||||
let lifetime = lifetime_source.as_ref().map(|source| source.retain());
|
||||
let (reply, result) = oneshot::channel();
|
||||
// Serialize admission and acceptance snapshots. Worker completion cannot
|
||||
// race acceptance, and cancelled waiters never alter this actual-job cap.
|
||||
{
|
||||
let mut accepted = self.tracking.accepted.lock().unwrap();
|
||||
if accepted.unfinished.len() >= self.max_jobs {
|
||||
return Err(Error::DbCapacityExceeded.into());
|
||||
}
|
||||
let id = accepted
|
||||
.last_job_id
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| E::from(Error::DbExecutorUnavailable))?;
|
||||
// Never reuse the identity of a rejected job: its destructor may run
|
||||
// after another submission has acquired the admission lock.
|
||||
accepted.last_job_id = id;
|
||||
let job = Job {
|
||||
operation: Box::new(move || {
|
||||
let outcome = catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| {
|
||||
report_unknown(id, "operation");
|
||||
Err(Error::DbOperationOutcomeUnknown.into())
|
||||
});
|
||||
let _ = reply.send(outcome);
|
||||
}),
|
||||
_lifetime: lifetime,
|
||||
lifetime_source,
|
||||
runtime,
|
||||
diagnostics,
|
||||
span,
|
||||
_completion: Completion {
|
||||
tracking: self.tracking.clone(),
|
||||
id,
|
||||
},
|
||||
};
|
||||
match sender.try_send(job) {
|
||||
Ok(()) => {
|
||||
accepted.unfinished.insert(id);
|
||||
}
|
||||
Err(error) => {
|
||||
// The rejected Job's completion also locks tracking; drop it only
|
||||
// after releasing the admission lock.
|
||||
drop(accepted);
|
||||
return Err(match error {
|
||||
mpsc::TrySendError::Full(_) => Error::DbCapacityExceeded,
|
||||
mpsc::TrySendError::Disconnected(_) => Error::DbExecutorUnavailable,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Wait for every job accepted before this barrier is first polled (not
|
||||
/// later submissions). This bypasses queue capacity and survives cancelled
|
||||
/// job waiters. It does not close the queue or stop new submissions.
|
||||
pub async fn drain(&self) -> Result<()> {
|
||||
if self.sender.is_none() {
|
||||
return Err(Error::DbExecutorUnavailable);
|
||||
}
|
||||
let through = self.tracking.accepted.lock().unwrap().last_job_id;
|
||||
loop {
|
||||
let notified = self.tracking.finished.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self
|
||||
.tracking
|
||||
.accepted
|
||||
.lock()
|
||||
.unwrap()
|
||||
.unfinished
|
||||
.first()
|
||||
.is_none_or(|id| *id > through)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rusqlite::{Connection, TransactionBehavior, ffi};
|
||||
use std::{ffi::c_void, time::Duration};
|
||||
|
||||
fn executor(queued: usize) -> DbExecutor {
|
||||
DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(queued).unwrap(),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn database() -> (tempfile::TempDir, Connection, Connection) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("jobs.sqlite");
|
||||
let owner = Connection::open(&path).unwrap();
|
||||
owner
|
||||
.execute_batch("CREATE TABLE writes (value INTEGER NOT NULL)")
|
||||
.unwrap();
|
||||
let worker = Connection::open(&path).unwrap();
|
||||
(dir, owner, worker)
|
||||
}
|
||||
|
||||
struct BusyGate {
|
||||
waiting: Option<oneshot::Sender<()>>,
|
||||
release: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
// A fixture-owned callback proves SQLite actually reached its lock wait.
|
||||
// Unlike busy_handler's fn pointer API, the SQLite API carries a per-call
|
||||
// context, so tests need neither process-global state nor guessed sleeps.
|
||||
struct BusyRegistration {
|
||||
db: *mut ffi::sqlite3,
|
||||
_gate: Box<BusyGate>,
|
||||
}
|
||||
|
||||
impl BusyRegistration {
|
||||
fn install(conn: &Connection, gate: BusyGate) -> Self {
|
||||
unsafe extern "C" fn wait(context: *mut c_void, _: i32) -> i32 {
|
||||
// The Box remains registered only while this connection and
|
||||
// BusyRegistration are alive, on this same dedicated worker.
|
||||
let gate = unsafe { &mut *context.cast::<BusyGate>() };
|
||||
if let Some(waiting) = gate.waiting.take() {
|
||||
let _ = waiting.send(());
|
||||
if gate.release.recv().is_ok() {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
let mut gate = Box::new(gate);
|
||||
// SAFETY: conn owns the handle; registration is dropped before conn.
|
||||
let db = unsafe { conn.handle() };
|
||||
let result = unsafe {
|
||||
ffi::sqlite3_busy_handler(db, Some(wait), (&mut *gate as *mut BusyGate).cast())
|
||||
};
|
||||
assert_eq!(result, ffi::SQLITE_OK);
|
||||
Self { db, _gate: gate }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BusyRegistration {
|
||||
fn drop(&mut self) {
|
||||
// SAFETY: unregister before releasing the callback's boxed context.
|
||||
unsafe {
|
||||
ffi::sqlite3_busy_handler(self.db, None, std::ptr::null_mut());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn observed<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::time::timeout(Duration::from_secs(5), future)
|
||||
.await
|
||||
.expect("DB job did not reach the expected observable state")
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_lock_wait_preserves_timer_and_independent_control_progress() {
|
||||
let (_dir, mut owner, mut worker) = database();
|
||||
let lock = owner
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let executor = executor(1);
|
||||
let (waiting, blocked) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut write = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _busy = BusyRegistration::install(
|
||||
&worker,
|
||||
BusyGate {
|
||||
waiting: Some(waiting),
|
||||
release: permit,
|
||||
},
|
||||
);
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
observed(blocked).await.unwrap();
|
||||
let control = tokio::spawn(async { 42 });
|
||||
assert_eq!(observed(control).await.unwrap(), 42);
|
||||
// The timer itself is the contract under test, not synchronization.
|
||||
observed(tokio::time::sleep(Duration::from_millis(1))).await;
|
||||
assert!(futures::poll!(&mut write).is_pending());
|
||||
lock.rollback().unwrap();
|
||||
release.send(()).unwrap();
|
||||
observed(write).await.unwrap();
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
struct Lease(Arc<std::sync::atomic::AtomicUsize>);
|
||||
struct Retained(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl DbOperationLifetime for Lease {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Box::new(Retained(self.0.clone()))
|
||||
}
|
||||
}
|
||||
impl Drop for Retained {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_mutation_retains_capacity_and_lease_until_sql_commits() {
|
||||
let (_dir, mut owner, mut worker) = database();
|
||||
let lock = owner
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let executor = executor(1);
|
||||
let (waiting, blocked) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut running = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _busy = BusyRegistration::install(
|
||||
&worker,
|
||||
BusyGate {
|
||||
waiting: Some(waiting),
|
||||
release: permit,
|
||||
},
|
||||
);
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut running).is_pending());
|
||||
observed(blocked).await.unwrap();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let mut queued_conn = Connection::open(_dir.path().join("jobs.sqlite")).unwrap();
|
||||
let mut queued = Box::pin(with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<()> {
|
||||
let tx = queued_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
// FIFO is observable in durable SQL, not inferred from waiter order.
|
||||
let first: i64 =
|
||||
tx.query_row("SELECT count(*) FROM writes WHERE value = 1", [], |row| {
|
||||
row.get(0)
|
||||
})?;
|
||||
assert_eq!(first, 1);
|
||||
tx.execute("INSERT INTO writes VALUES (2)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
assert!(matches!(
|
||||
executor.run(|| Ok::<_, Error>(())).await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
let mut drained = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
lock.rollback().unwrap();
|
||||
release.send(()).unwrap();
|
||||
observed(running).await.unwrap();
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn post_commit_child_job_retains_admitted_lifetime_after_parent_waiter_and_job_finish() {
|
||||
let (dir, owner, mut parent_conn) = database();
|
||||
let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap();
|
||||
let parent = executor(1);
|
||||
let child = executor(1);
|
||||
let child_owner = child.clone();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let source: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let (entered, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let (child_entered, child_running) = oneshot::channel();
|
||||
let (finish_child, child_permit) = mpsc::channel();
|
||||
let mut operation = Box::pin(with_operation_lifetime(
|
||||
source,
|
||||
parent.run(move || -> Result<()> {
|
||||
let _ = entered.send(());
|
||||
let _ = permit.recv();
|
||||
let tx = parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
child_owner.dispatch(move || {
|
||||
let tx = child_conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let committed: i64 = tx
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(committed, 1, "child must see its parent's committed SQL");
|
||||
let _ = child_entered.send(());
|
||||
let _ = child_permit.recv();
|
||||
tx.execute("INSERT INTO writes VALUES (2)", []).unwrap();
|
||||
tx.commit().unwrap();
|
||||
})?;
|
||||
Ok(())
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(operation);
|
||||
release.send(()).unwrap();
|
||||
observed(child_running).await.unwrap();
|
||||
observed(parent.drain()).await.unwrap();
|
||||
assert_eq!(
|
||||
count.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"accepted continuation lost authority when its parent finished"
|
||||
);
|
||||
let mut drained = Box::pin(child.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
finish_child.send(()).unwrap();
|
||||
observed(drained).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let error = executor
|
||||
.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.execute("INSERT INTO writes VALUES (NULL)", [])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::Sqlite(_)));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panic_after_commit_reports_unknown_without_retry_and_worker_remains_available() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let error = executor
|
||||
.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
panic!("lost outcome after commit");
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(executor.run(|| Ok::<_, Error>(42)).await.unwrap(), 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panic_before_commit_rolls_back_and_releases_lease_and_capacity() {
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let error = with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<()> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
panic!("lost outcome before commit");
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, Error::DbOperationOutcomeUnknown));
|
||||
executor.drain().await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
struct DiscardedReplyHook {
|
||||
expected_runtime: tokio::runtime::Id,
|
||||
lease_count: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Option<oneshot::Sender<(bool, usize)>>,
|
||||
}
|
||||
|
||||
impl Drop for DiscardedReplyHook {
|
||||
fn drop(&mut self) {
|
||||
let runtime =
|
||||
tokio::runtime::Handle::try_current().expect("reply cleanup lost runtime context");
|
||||
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
|
||||
let same_runtime = runtime.id() == self.expected_runtime;
|
||||
let dropped = self.dropped.take().unwrap();
|
||||
tokio::spawn(async move {
|
||||
let _ = dropped.send((same_runtime, retained));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_queued_sql_keeps_runtime_context_and_lease_through_post_commit_and_reply_drop()
|
||||
{
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut blocker).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
let expected_runtime = tokio::runtime::Handle::current().id();
|
||||
let (hook, published) = oneshot::channel();
|
||||
let (finish, finishing) = mpsc::channel();
|
||||
let (dropped, reply_dropped) = oneshot::channel();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let operation_count = count.clone();
|
||||
let mut queued = Box::pin(with_operation_lifetime(
|
||||
lease,
|
||||
executor.run(move || -> Result<_> {
|
||||
let tx = worker.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
tx.execute("INSERT INTO writes VALUES (1)", [])?;
|
||||
tx.commit()?;
|
||||
// Model synchronous Host hooks: publish only after commit, checking
|
||||
// try_current and spawning without driving any future on this worker.
|
||||
let runtime = tokio::runtime::Handle::try_current()
|
||||
.expect("post-commit hook lost runtime context");
|
||||
assert_eq!(runtime.id(), expected_runtime);
|
||||
tokio::spawn(async move {
|
||||
let _ = hook.send(());
|
||||
});
|
||||
let _ = finishing.recv();
|
||||
Ok(DiscardedReplyHook {
|
||||
expected_runtime,
|
||||
lease_count: operation_count,
|
||||
dropped: Some(dropped),
|
||||
})
|
||||
}),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
release.send(()).unwrap();
|
||||
observed(blocker).await.unwrap();
|
||||
observed(published).await.unwrap();
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
finish.send(()).unwrap();
|
||||
assert_eq!(observed(reply_dropped).await.unwrap(), (true, 1));
|
||||
observed(drain).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn multiple_workers_bound_actual_jobs_even_after_all_waiters_are_cancelled() {
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(2).unwrap(),
|
||||
queued: NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let mut releases = Vec::new();
|
||||
// Observe each worker executing before submitting the next operation,
|
||||
// so queue occupancy and overload are deterministic, not timing guesses.
|
||||
for _ in 0..2 {
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut operation = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(operation);
|
||||
releases.push(release);
|
||||
}
|
||||
let (finished, mut queued_finished) = oneshot::channel();
|
||||
let mut queued = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = finished.send(());
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
drop(queued);
|
||||
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
for _ in 0..8 {
|
||||
let rejected_runs = rejected_runs.clone();
|
||||
assert!(matches!(
|
||||
executor
|
||||
.run(move || {
|
||||
rejected_runs.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok::<_, Error>(())
|
||||
})
|
||||
.await,
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
}
|
||||
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert!(matches!(
|
||||
queued_finished.try_recv(),
|
||||
Err(oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
for release in releases {
|
||||
release.send(()).unwrap();
|
||||
}
|
||||
observed(drain).await.unwrap();
|
||||
observed(queued_finished).await.unwrap();
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap();
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct UnknownDiagnostic {
|
||||
code: Option<String>,
|
||||
phase: Option<String>,
|
||||
accepted_job_id: Option<u64>,
|
||||
}
|
||||
|
||||
impl tracing::field::Visit for UnknownDiagnostic {
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
match field.name() {
|
||||
"code" => self.code = Some(value.to_owned()),
|
||||
"phase" => self.phase = Some(value.to_owned()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
|
||||
if field.name() == "accepted_job_id" {
|
||||
self.accepted_job_id = Some(value);
|
||||
}
|
||||
}
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct DiagnosticCapture(Arc<Mutex<Vec<UnknownDiagnostic>>>);
|
||||
|
||||
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for DiagnosticCapture {
|
||||
fn on_event(
|
||||
&self,
|
||||
event: &tracing::Event<'_>,
|
||||
_: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let mut diagnostic = UnknownDiagnostic::default();
|
||||
event.record(&mut diagnostic);
|
||||
self.0.lock().unwrap().push(diagnostic);
|
||||
}
|
||||
}
|
||||
|
||||
struct DispatchCleanup {
|
||||
expected_runtime: tokio::runtime::Id,
|
||||
lease_count: Arc<std::sync::atomic::AtomicUsize>,
|
||||
started: Option<oneshot::Sender<(bool, usize)>>,
|
||||
release: mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
impl Drop for DispatchCleanup {
|
||||
fn drop(&mut self) {
|
||||
// No panicking assertions during unwind: report observations to the
|
||||
// test, and gate cleanup so drain/lifetime behavior is observable.
|
||||
let same_runtime = tokio::runtime::Handle::try_current()
|
||||
.is_ok_and(|runtime| runtime.id() == self.expected_runtime);
|
||||
let retained = self.lease_count.load(std::sync::atomic::Ordering::SeqCst);
|
||||
if let Some(started) = self.started.take() {
|
||||
let _ = started.send((same_runtime, retained));
|
||||
}
|
||||
let _ = self.release.recv();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn dispatched_panic_is_bounded_retained_through_cleanup_diagnosed_and_drained_without_retry()
|
||||
{
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
let (_dir, owner, mut worker) = database();
|
||||
let executor = executor(1);
|
||||
let (started, running) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut blocker = Box::pin(executor.run(move || -> Result<()> {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv();
|
||||
Ok(())
|
||||
}));
|
||||
assert!(futures::poll!(&mut blocker).is_pending());
|
||||
observed(running).await.unwrap();
|
||||
drop(blocker); // Cancellation does not free its running slot.
|
||||
let expected_runtime = tokio::runtime::Handle::current().id();
|
||||
let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let lease: Arc<dyn DbOperationLifetime> = Arc::new(Lease(count.clone()));
|
||||
let operation_count = count.clone();
|
||||
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let operation_attempts = attempts.clone();
|
||||
let (cleanup_started, cleaning_up) = oneshot::channel();
|
||||
let (finish, finishing) = mpsc::channel();
|
||||
// Model multiple tracing contexts without setting a global subscriber.
|
||||
// This also prevents tracing's single-subscriber callsite optimization
|
||||
// from caching an unsubscribed parallel test's Interest::never here.
|
||||
let _other_diagnostics = tracing::Dispatch::new(tracing_subscriber::registry());
|
||||
let captured = DiagnosticCapture::default();
|
||||
let diagnostics =
|
||||
tracing::Dispatch::new(tracing_subscriber::registry().with(captured.clone()));
|
||||
with_operation_lifetime(lease, async {
|
||||
tracing::dispatcher::with_default(&diagnostics, || {
|
||||
executor.dispatch(move || {
|
||||
let _cleanup = DispatchCleanup {
|
||||
expected_runtime,
|
||||
lease_count: operation_count,
|
||||
started: Some(cleanup_started),
|
||||
release: finishing,
|
||||
};
|
||||
operation_attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let tx = worker
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
tx.execute("INSERT INTO writes VALUES (1)", []).unwrap();
|
||||
tx.commit().unwrap();
|
||||
panic!("dispatched post-commit outcome lost");
|
||||
})
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(); // Dispatch is admitted synchronously; no waiter survives.
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
let rejected_runs = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let rejected_operation = rejected_runs.clone();
|
||||
assert!(matches!(
|
||||
executor.dispatch(move || {
|
||||
rejected_operation.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}),
|
||||
Err(Error::DbCapacityExceeded)
|
||||
));
|
||||
release.send(()).unwrap();
|
||||
assert_eq!(observed(cleaning_up).await.unwrap(), (true, 1));
|
||||
assert_eq!(
|
||||
owner
|
||||
.query_row("SELECT count(*) FROM writes", [], |row| row
|
||||
.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
let mut drain = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drain).is_pending());
|
||||
finish.send(()).unwrap();
|
||||
observed(drain).await.unwrap();
|
||||
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
assert_eq!(rejected_runs.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
{
|
||||
let events = captured.0.lock().unwrap();
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(
|
||||
events[0].code.as_deref(),
|
||||
Some("db_operation_outcome_unknown")
|
||||
);
|
||||
assert_eq!(events[0].phase.as_deref(), Some("operation"));
|
||||
assert!(events[0].accepted_job_id.is_some());
|
||||
}
|
||||
let (completed, completion) = oneshot::channel();
|
||||
executor
|
||||
.dispatch(move || {
|
||||
let _ = completed.send(());
|
||||
})
|
||||
.unwrap();
|
||||
observed(executor.drain()).await.unwrap();
|
||||
observed(completion).await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -443,6 +443,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let job = store
|
||||
@@ -491,6 +492,7 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let job = store
|
||||
|
||||
@@ -469,6 +469,7 @@ fn migrated_workdir_request_replays_and_retries_without_accepting_changed_displa
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
};
|
||||
let migrated = store
|
||||
@@ -652,6 +653,7 @@ fn deleted_successful_workdir_archives_verbatim_history_without_authorizing_retr
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let archived = store
|
||||
@@ -1027,6 +1029,7 @@ fn migrated_ssh_mutations_replay_complete_old_input_without_reexecution_or_recei
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let service =
|
||||
@@ -1179,6 +1182,7 @@ fn deleted_legacy_receipt_does_not_bind_to_later_reused_key_ordinal() {
|
||||
// The service consumes current receipt columns; unrelated schemas are not this fixture.
|
||||
rename_saved_secret_receipt_counters(&conn).unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
});
|
||||
let service =
|
||||
|
||||
@@ -533,6 +533,7 @@ mod existing_workspace_provisioning {
|
||||
// This fixture contains only the schema-87 cutover authorities, not the
|
||||
// unrelated full Server schema. Exercise the real service/store methods.
|
||||
Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
})
|
||||
}
|
||||
@@ -644,6 +645,7 @@ mod existing_workspace_provisioning {
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(&path).unwrap())),
|
||||
});
|
||||
let service = WorkspaceSigningIdentityService::new(
|
||||
@@ -816,7 +818,7 @@ mod current_authority {
|
||||
use super::*;
|
||||
|
||||
fn binding_store() -> (SqliteWorkspaceStore, WorkspaceRuntimeBinding) {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
let runtime_key =
|
||||
worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-key").unwrap();
|
||||
store.with_conn(|conn| {
|
||||
|
||||
@@ -23,7 +23,10 @@ use std::{
|
||||
};
|
||||
use workdir::{
|
||||
Workdir, WorkdirError, WorkdirSessionHandle,
|
||||
http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization},
|
||||
http::{
|
||||
OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization,
|
||||
WorkdirHttpAuthorizationFuture,
|
||||
},
|
||||
};
|
||||
use worker_runtime::RuntimeWorkspaceScope;
|
||||
use worker_runtime::catalog::{
|
||||
@@ -1294,8 +1297,8 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
fn observation_source(
|
||||
&self,
|
||||
_worker_id: &str,
|
||||
) -> Option<crate::observation::RuntimeObservationSource> {
|
||||
None
|
||||
) -> Result<Option<crate::observation::RuntimeObservationSource>, RuntimeDiagnostic> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn send_input(&self, worker_id: &str, _request: WorkerInputRequest) -> WorkerInputResult {
|
||||
@@ -2233,6 +2236,11 @@ impl RuntimeRegistry {
|
||||
let runtime = self.runtime(runtime_id)?;
|
||||
runtime
|
||||
.observation_source(worker_id)
|
||||
.map_err(|diagnostic| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: runtime_id.to_owned(),
|
||||
code: diagnostic.code,
|
||||
message: diagnostic.message,
|
||||
})?
|
||||
.ok_or_else(|| RuntimeRegistryError::UnknownWorker {
|
||||
worker: worker.clone(),
|
||||
})
|
||||
@@ -3200,17 +3208,21 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
fn observation_source(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
) -> Option<crate::observation::RuntimeObservationSource> {
|
||||
let worker_ref = self.worker_ref(worker_id)?;
|
||||
) -> Result<Option<crate::observation::RuntimeObservationSource>, RuntimeDiagnostic> {
|
||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if self.runtime.worker_detail(&worker_ref).is_err() {
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
Some(crate::observation::RuntimeObservationSource::embedded(
|
||||
crate::observation::EmbeddedRuntimeObservationSource {
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
|
||||
runtime: self.runtime.clone(),
|
||||
worker_ref,
|
||||
},
|
||||
Ok(Some(
|
||||
crate::observation::RuntimeObservationSource::embedded(
|
||||
crate::observation::EmbeddedRuntimeObservationSource {
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
|
||||
runtime: self.runtime.clone(),
|
||||
worker_ref,
|
||||
},
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -3477,6 +3489,38 @@ impl WorkspaceRuntimeAuthorization {
|
||||
}
|
||||
}
|
||||
|
||||
/// Offload the complete live-binding check and request signing operation.
|
||||
/// Claims are minted inside the bounded job, not cached before admission.
|
||||
pub(crate) async fn issue_async(
|
||||
&self,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
operation: String,
|
||||
worker_id: Option<String>,
|
||||
body: Vec<u8>,
|
||||
) -> Result<String, RuntimeDiagnostic> {
|
||||
let executor = self.store.db_executor();
|
||||
let authorization = self.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
Ok::<_, Error>(authorization.issue(
|
||||
&method,
|
||||
&path_and_query,
|
||||
&operation,
|
||||
worker_id.as_deref(),
|
||||
&body,
|
||||
))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
diagnostic(
|
||||
"workspace_runtime_authorization_unavailable",
|
||||
HostDiagnosticSeverity::Error,
|
||||
error.to_string(),
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
pub(crate) fn issue(
|
||||
&self,
|
||||
method: &str,
|
||||
@@ -3649,25 +3693,24 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization {
|
||||
impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization {
|
||||
fn bearer_token(
|
||||
&self,
|
||||
method: &str,
|
||||
path_and_query: &str,
|
||||
body: &[u8],
|
||||
) -> Result<String, WorkdirError> {
|
||||
if let Some(authorization) = &self.workspace_authorization {
|
||||
return authorization
|
||||
.issue(
|
||||
method,
|
||||
path_and_query,
|
||||
workspace_runtime_operation(method, path_and_query),
|
||||
None,
|
||||
body,
|
||||
method: String,
|
||||
path_and_query: String,
|
||||
body: Vec<u8>,
|
||||
) -> WorkdirHttpAuthorizationFuture {
|
||||
let authorization = self.clone();
|
||||
Box::pin(async move {
|
||||
if let Some(authorizer) = &authorization.workspace_authorization {
|
||||
let operation = workspace_runtime_operation(&method, &path_and_query).to_string();
|
||||
return authorizer
|
||||
.issue_async(method, path_and_query, operation, None, body)
|
||||
.await
|
||||
.map_err(|error| WorkdirError::Unavailable(error.message));
|
||||
}
|
||||
authorization.fallback_bearer_token.ok_or_else(|| {
|
||||
WorkdirError::Unavailable(
|
||||
"remote Runtime does not have bearer authorization configured".to_string(),
|
||||
)
|
||||
.map_err(|error| WorkdirError::Unavailable(error.message));
|
||||
}
|
||||
self.fallback_bearer_token.clone().ok_or_else(|| {
|
||||
WorkdirError::Unavailable(
|
||||
"remote Runtime does not have bearer authorization configured".to_string(),
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5290,20 +5333,22 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
fn observation_source(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
) -> Option<crate::observation::RuntimeObservationSource> {
|
||||
) -> Result<Option<crate::observation::RuntimeObservationSource>, RuntimeDiagnostic> {
|
||||
let path = format!("/v1/workers/{worker_id}/protocol/ws");
|
||||
let bearer_token = match &self.workspace_authorization {
|
||||
Some(authorization) => authorization
|
||||
.issue("GET", &path, "workers:protocol", Some(worker_id), &[])
|
||||
.ok(),
|
||||
Some(authorization) => {
|
||||
Some(authorization.issue("GET", &path, "workers:protocol", Some(worker_id), &[])?)
|
||||
}
|
||||
None => self.bearer_token.clone(),
|
||||
};
|
||||
Some(crate::observation::RuntimeObservationSource::remote_ws(
|
||||
crate::observation::RuntimeObservationSourceConfig {
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
|
||||
endpoint: self.ws_endpoint(worker_id),
|
||||
bearer_token,
|
||||
},
|
||||
Ok(Some(
|
||||
crate::observation::RuntimeObservationSource::remote_ws(
|
||||
crate::observation::RuntimeObservationSourceConfig {
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
|
||||
endpoint: self.ws_endpoint(worker_id),
|
||||
bearer_token,
|
||||
},
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -6348,6 +6393,79 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
|
||||
#[tokio::test]
|
||||
async fn async_workdir_authorization_uses_store_capacity_without_bearer_fallback() {
|
||||
use crate::db_executor::{DbExecutor, DbExecutorLimits};
|
||||
use crate::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding};
|
||||
use crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore;
|
||||
|
||||
let executor = DbExecutor::new(DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::in_memory_for_test()
|
||||
.unwrap()
|
||||
.with_db_executor(executor.clone()),
|
||||
);
|
||||
let service = WorkspaceSigningIdentityService::new(
|
||||
store.clone(),
|
||||
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
|
||||
);
|
||||
let authorization = RemoteWorkdirAuthorization {
|
||||
workspace_authorization: Some(WorkspaceRuntimeAuthorization::new(
|
||||
store,
|
||||
service,
|
||||
"https://backend.test",
|
||||
WorkspaceRuntimeBinding {
|
||||
workspace_id: "workspace".into(),
|
||||
runtime_id: "runtime".into(),
|
||||
display_name: "Runtime".into(),
|
||||
base_url: "https://runtime.test".into(),
|
||||
public_key_fingerprint: "unused".into(),
|
||||
public_key: "unused".into(),
|
||||
binding_id: "binding".into(),
|
||||
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
revoked_at: None,
|
||||
},
|
||||
)),
|
||||
fallback_bearer_token: Some("must-not-be-used".into()),
|
||||
};
|
||||
|
||||
// Hold the running slot and fill the only queue slot. The callback must
|
||||
// reject admission, not run inline or silently switch authentication.
|
||||
let (release, wait) = std::sync::mpsc::channel::<()>();
|
||||
let (started, running) = tokio::sync::oneshot::channel();
|
||||
executor
|
||||
.dispatch(move || {
|
||||
let _ = started.send(());
|
||||
let _ = wait.recv_timeout(Duration::from_secs(5));
|
||||
})
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), running)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
executor.dispatch(|| {}).unwrap();
|
||||
let error = authorization
|
||||
.bearer_token(
|
||||
"POST".into(),
|
||||
"/v1/workdir-sessions/s/operations".into(),
|
||||
b"{}".to_vec(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string()
|
||||
));
|
||||
drop(release);
|
||||
executor.drain().await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ticket_worker_spawn_intent_uses_worker_wire_role() {
|
||||
let intent = WorkerSpawnIntent::TicketRole {
|
||||
|
||||
@@ -32,7 +32,8 @@ struct Fixture {
|
||||
impl Fixture {
|
||||
async fn new() -> Self {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap());
|
||||
let store =
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(temp.path().join("server.db")).unwrap());
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".into(),
|
||||
|
||||
@@ -45,7 +45,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() {
|
||||
let artifacts = evidence_rows(&conn, "artifacts");
|
||||
let receipts = evidence_rows(&conn, "repository_secret_legacy_receipts");
|
||||
drop(conn);
|
||||
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
upgraded
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
|
||||
@@ -73,7 +73,7 @@ fn evidence_column_cutover_preserves_all_values_and_current_schema_names() {
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
for store in [upgraded, SqliteWorkspaceStore::in_memory().unwrap()] {
|
||||
for store in [upgraded, SqliteWorkspaceStore::in_memory_for_test().unwrap()] {
|
||||
store.with_conn(|conn| {
|
||||
let mut stmt = conn.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")?;
|
||||
let objects = stmt.query_map([],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
|
||||
@@ -10,6 +10,7 @@ pub mod backend_job;
|
||||
pub mod companion;
|
||||
pub mod config;
|
||||
pub mod config_source;
|
||||
pub mod db_executor;
|
||||
pub mod feature_storage;
|
||||
pub mod hosts;
|
||||
pub mod memory_backend;
|
||||
@@ -159,6 +160,12 @@ pub enum Error {
|
||||
WorkspaceSigningIdentity { code: String, message: String },
|
||||
#[error("Workspace Drive error: {0}")]
|
||||
Drive(#[from] workspace_drive::Error),
|
||||
#[error("DB execution capacity exceeded")]
|
||||
DbCapacityExceeded,
|
||||
#[error("DB executor unavailable")]
|
||||
DbExecutorUnavailable,
|
||||
#[error("DB operation outcome unknown; do not automatically retry")]
|
||||
DbOperationOutcomeUnknown,
|
||||
#[error("store error: {0}")]
|
||||
Store(String),
|
||||
}
|
||||
|
||||
@@ -217,7 +217,7 @@ mod tests {
|
||||
async fn authority() -> (TempDir, SqliteWorkspaceAuthority) {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let db_path = temp.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open(&db_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&db_path).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-test".to_string(),
|
||||
|
||||
@@ -32,7 +32,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() {
|
||||
drop(conn);
|
||||
for attempt in 0..3 {
|
||||
let error = match attempt {
|
||||
0 => SqliteWorkspaceStore::open(&path).err().unwrap(),
|
||||
0 => SqliteWorkspaceStore::open_for_test(&path).err().unwrap(),
|
||||
1 => SqliteWorkspaceStore::migration_plan(&path).unwrap_err(),
|
||||
_ => SqliteWorkspaceStore::migrate_database(&path).unwrap_err(),
|
||||
};
|
||||
@@ -62,7 +62,7 @@ fn opening_schema_84_with_uncertain_removal_preserves_old_server_recovery() {
|
||||
)
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
|
||||
assert_eq!(conn.query_row("SELECT input_fingerprint FROM worker_removal_operations WHERE operation_id='removal'",[],|r|r.get::<_,String>(0))?,"legacy-receipt");
|
||||
@@ -109,7 +109,7 @@ fn migration_dry_run_and_apply_share_the_original_repository_secret_authority()
|
||||
assert!(!table_exists(&conn, "server_secret_objects").unwrap());
|
||||
drop(conn);
|
||||
assert_eq!(SqliteWorkspaceStore::migrate_database(&path).unwrap(), plan);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?,LATEST_SCHEMA_VERSION);
|
||||
let (nonce, mut sealed): (Vec<u8>,Vec<u8>) = conn.query_row(
|
||||
|
||||
@@ -2290,7 +2290,7 @@ mod tests {
|
||||
) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let database = dir.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
|
||||
let timestamp = now();
|
||||
for workspace_id in ["workspace-a", "workspace-b"] {
|
||||
futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord {
|
||||
|
||||
@@ -1116,7 +1116,7 @@ mod tests {
|
||||
WorkerId::from_legacy_u64(1)
|
||||
}
|
||||
fn setup() -> SqliteWorkspaceStore {
|
||||
let s = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let s = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
s.with_conn(|c| {
|
||||
c.execute(
|
||||
"INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) \
|
||||
|
||||
@@ -965,7 +965,7 @@ async fn connect_runtime(
|
||||
let mut request = endpoint
|
||||
.into_client_request()
|
||||
.map_err(|error| format!("invalid Runtime subscription endpoint: {error}"))?;
|
||||
if let Some(token) = runtime_token(config, workspace_id)? {
|
||||
if let Some(token) = runtime_token(config, workspace_id).await? {
|
||||
request.headers_mut().insert(
|
||||
"authorization",
|
||||
format!("Bearer {token}")
|
||||
@@ -1021,13 +1021,20 @@ fn runtime_endpoint(base_url: &str) -> String {
|
||||
format!("{base}/v1/protocol/ws")
|
||||
}
|
||||
}
|
||||
fn runtime_token(
|
||||
async fn runtime_token(
|
||||
config: &RemoteRuntimeConfig,
|
||||
_workspace_id: &str,
|
||||
) -> Result<Option<String>, String> {
|
||||
if let Some(authorization) = config.workspace_authorization.as_ref() {
|
||||
return authorization
|
||||
.issue("GET", "/v1/protocol/ws", "workers:list", None, &[])
|
||||
.issue_async(
|
||||
"GET".to_string(),
|
||||
"/v1/protocol/ws".to_string(),
|
||||
"workers:list".to_string(),
|
||||
None,
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(|error| error.message);
|
||||
}
|
||||
|
||||
@@ -38,8 +38,8 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("upgraded.db");
|
||||
prepare_retained_schema(&path, 85);
|
||||
let upgraded = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
let mut mismatches = Vec::new();
|
||||
for table in [
|
||||
"workspace_tags",
|
||||
|
||||
+6446
-3875
File diff suppressed because it is too large
Load Diff
@@ -65,70 +65,85 @@ pub(super) fn require_candidate_grant(
|
||||
/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface
|
||||
/// background work; the new Job regenerates it and does not infer surface success.
|
||||
pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult<bool> {
|
||||
let job_api = api.clone();
|
||||
let subject_id = subject_id.to_owned();
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let Some(owner) = api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
else {
|
||||
return Ok(true);
|
||||
let (worker, complete) = api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
let api = &job_api;
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let Some(owner) = api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
else {
|
||||
return Ok((None, true));
|
||||
};
|
||||
let mut summary = api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?;
|
||||
summary.singleton_key = Some(owner.key.clone());
|
||||
require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?;
|
||||
let stopped = summary.state.eq_ignore_ascii_case("stopped");
|
||||
let idle = summary.state.eq_ignore_ascii_case("idle")
|
||||
&& summary
|
||||
.worker_state
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
|
||||
if !stopped && !idle {
|
||||
return Ok((None, false));
|
||||
}
|
||||
if api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.as_ref()
|
||||
.map(|o| &o.worker)
|
||||
!= Some(&owner.worker)
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"legacy consolidation owner moved during cutover".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if !summary.state.eq_ignore_ascii_case("stopped") {
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(
|
||||
&owner.worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(
|
||||
"Retire legacy subjektiv consolidator after foreground completion"
|
||||
.into(),
|
||||
),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok((None, false));
|
||||
}
|
||||
}
|
||||
if !api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?
|
||||
.state
|
||||
.eq_ignore_ascii_case("stopped")
|
||||
{
|
||||
return Ok((None, false));
|
||||
}
|
||||
Ok::<_, ApiError>((Some(owner.worker), false))
|
||||
})
|
||||
.await?;
|
||||
let Some(worker) = worker else {
|
||||
return Ok(complete);
|
||||
};
|
||||
let mut summary = api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?;
|
||||
summary.singleton_key = Some(owner.key.clone());
|
||||
require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?;
|
||||
let stopped = summary.state.eq_ignore_ascii_case("stopped");
|
||||
let idle = summary.state.eq_ignore_ascii_case("idle")
|
||||
&& summary
|
||||
.worker_state
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
|
||||
if !stopped && !idle {
|
||||
return Ok(false);
|
||||
}
|
||||
if api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.as_ref()
|
||||
.map(|o| &o.worker)
|
||||
!= Some(&owner.worker)
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"legacy consolidation owner moved during cutover".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if !summary.state.eq_ignore_ascii_case("stopped") {
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(
|
||||
&owner.worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(
|
||||
"Retire legacy subjektiv consolidator after foreground completion".into(),
|
||||
),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
if !api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?
|
||||
.state
|
||||
.eq_ignore_ascii_case("stopped")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let removal = WorkerRemovalService::new(api)
|
||||
.execute_target_removal(
|
||||
api.runtime.as_ref(),
|
||||
&owner.worker,
|
||||
&worker,
|
||||
"subjektiv consolidation Job cutover",
|
||||
None,
|
||||
)
|
||||
@@ -141,10 +156,17 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -
|
||||
))
|
||||
.into());
|
||||
}
|
||||
Ok(api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.is_none())
|
||||
let store = api.store.clone();
|
||||
let workspace_id = api.workspace_id().to_owned();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store
|
||||
.current_worker_singleton_owner(&workspace_id, &key)
|
||||
.map(|owner| owner.is_none())
|
||||
.map_err(ApiError::from)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn bounded_candidate_batch(
|
||||
|
||||
@@ -806,7 +806,19 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc
|
||||
.unwrap();
|
||||
};
|
||||
let ((status, body, events), ()) = tokio::join!(
|
||||
request_with_logs(build_router(fixture.api.clone()), request),
|
||||
async {
|
||||
let result = request_with_logs(build_router(fixture.api.clone()), request).await;
|
||||
// A pre-dispatch rejection leaves the fake provider waiting on an
|
||||
// open channel. Fail with the actual response here instead of
|
||||
// hanging join! until the outer suite timeout hides that cause.
|
||||
assert_eq!(
|
||||
result.0,
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"early command-start response: {}",
|
||||
result.1
|
||||
);
|
||||
result
|
||||
},
|
||||
provider
|
||||
);
|
||||
assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}");
|
||||
@@ -832,3 +844,78 @@ async fn self_workdir_post_start_registration_failure_does_not_claim_pre_dispatc
|
||||
"post-dispatch failure was retried"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_host_before_command_start_returns_availability_without_a_provider_operation() {
|
||||
let mut fixture = manual_worker_assignment_fixture().await;
|
||||
let identity =
|
||||
worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id)
|
||||
.unwrap();
|
||||
configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id);
|
||||
let (sender, mut commands) = tokio::sync::mpsc::channel(1);
|
||||
let connection = Arc::new(ExternalProviderConnection {
|
||||
grant_id: "overload-grant".into(),
|
||||
workdir_id: fixture.main_workdir_id.clone(),
|
||||
provider_instance_id: "overload-provider".into(),
|
||||
generation: 1,
|
||||
expires_at: None,
|
||||
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||
admission: Arc::new(tokio::sync::Semaphore::new(1)),
|
||||
shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
sender,
|
||||
});
|
||||
fixture.runtime.workdir_session_factory = Some(Arc::new(move |_| {
|
||||
Ok(Arc::new(ExternalProviderWorkdirSession::new(
|
||||
connection.clone(),
|
||||
None,
|
||||
)) as WorkdirSessionHandle)
|
||||
}));
|
||||
fixture
|
||||
.api
|
||||
.runtime
|
||||
.register_or_replace(fixture.runtime.clone());
|
||||
let host = fixture.api.host_executors.host().unwrap();
|
||||
host.drain().await.unwrap();
|
||||
let mut releases = Vec::new(); // dropping senders also releases jobs on panic
|
||||
for _ in 0..2 {
|
||||
let (release, permit) = std::sync::mpsc::channel::<()>();
|
||||
releases.push(release);
|
||||
let (started, accepted) = tokio::sync::oneshot::channel();
|
||||
host.dispatch(move || {
|
||||
let _ = started.send(());
|
||||
let _ = permit.recv_timeout(std::time::Duration::from_secs(10));
|
||||
})
|
||||
.unwrap();
|
||||
accepted.await.unwrap();
|
||||
}
|
||||
for _ in 0..16 {
|
||||
host.dispatch(|| {}).unwrap();
|
||||
}
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations");
|
||||
let request = runtime_source_request(
|
||||
&identity,
|
||||
Some(&fixture.worker.worker_id),
|
||||
"POST",
|
||||
&path,
|
||||
serde_json::to_vec(&json!({
|
||||
"target_workdir": "checkout",
|
||||
"operation": {"operation": "command_start", "request": {
|
||||
"command": "not-dispatched", "timeout_secs": 1, "output_limit": 1024,
|
||||
"cwd": "", "spill_dir": null, "tool_call_id": "not-dispatched"
|
||||
}}
|
||||
}))
|
||||
.unwrap(),
|
||||
);
|
||||
let (status, body, _) = request_with_logs(build_router(fixture.api.clone()), request).await;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");
|
||||
assert!(body.to_string().contains("db_capacity_exceeded"), "{body}");
|
||||
// The response is already final but the live fake-provider receiver remains
|
||||
// Pending. An unbounded join of those two futures hid this causal rejection
|
||||
// behind the original 600-second suite timeout. No process or command ran.
|
||||
let provider_command = commands.recv();
|
||||
tokio::pin!(provider_command);
|
||||
assert!(futures::poll!(provider_command.as_mut()).is_pending());
|
||||
drop(releases);
|
||||
host.drain().await.unwrap();
|
||||
fixture.api.store.db_executor().drain().await.unwrap();
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
|
||||
configure_runtime_request_auth(&mut api, &identity, "runtime-test");
|
||||
let revoked = RuntimeIdentityMaterial::generate("revoked-runtime-secret").unwrap();
|
||||
configure_runtime_request_auth(&mut api, &revoked, "revoked-runtime-secret");
|
||||
SqliteWorkspaceStore::open(&api.config.database_path)
|
||||
SqliteWorkspaceStore::open_for_test(&api.config.database_path)
|
||||
.unwrap()
|
||||
.revoke_workspace_runtime_binding(
|
||||
TEST_WORKSPACE_ID,
|
||||
@@ -70,7 +70,7 @@ async fn rejected_runtime_proofs_log_reason_and_verification_without_secrets_on_
|
||||
let body = "body-secret";
|
||||
let apps = [
|
||||
build_router(api.clone()),
|
||||
workspace_server_router(WorkspaceServerApi::new(
|
||||
workspace_server_router(WorkspaceServerApi::new_for_test(
|
||||
api.config.clone(),
|
||||
api.store.clone(),
|
||||
)),
|
||||
@@ -318,7 +318,7 @@ async fn runtime_proof_setup_failure_logs_no_internal_error_and_bounds_request_m
|
||||
&error,
|
||||
)
|
||||
});
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
@@ -347,7 +347,7 @@ async fn repository_runtime_proof_401_keeps_generic_envelope_on_both_routers() {
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/repositories");
|
||||
for app in [
|
||||
build_router(api.clone()),
|
||||
workspace_server_router(WorkspaceServerApi::new(
|
||||
workspace_server_router(WorkspaceServerApi::new_for_test(
|
||||
api.config.clone(),
|
||||
api.store.clone(),
|
||||
)),
|
||||
|
||||
@@ -40,7 +40,7 @@ async fn cached_drive_handle_rejects_mutation_after_deletion_reservation() {
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let api = WorkspaceApi::new(config.clone(), store.clone())
|
||||
let api = WorkspaceApi::new_for_test(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let cached = api.drive().await.unwrap();
|
||||
@@ -104,7 +104,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature
|
||||
// Use a separate embedded-runtime namespace for the first API: this test
|
||||
// reconstructs Drive authority in-process rather than spawning/restarting
|
||||
// product processes (runtime observers may retain their namespace).
|
||||
let first_api = WorkspaceApi::new(config.clone(), store.clone())
|
||||
let first_api = WorkspaceApi::new_for_test(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
// Merely constructing a Workspace API must not create blob storage.
|
||||
@@ -160,7 +160,7 @@ async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
drop(first_api);
|
||||
let recovered = WorkspaceServerApi::new(config.clone(), store.clone());
|
||||
let recovered = WorkspaceServerApi::new_for_test(config.clone(), store.clone());
|
||||
let api = recovered
|
||||
.api_for_workspace(&config.workspace_id)
|
||||
.await
|
||||
|
||||
@@ -39,10 +39,10 @@ impl RealDriveFixture {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(temp.path());
|
||||
config.repositories.clear();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
let owner = seed_test_api_token(store.as_ref(), "drive-web-owner");
|
||||
let second = seed_test_api_token(store.as_ref(), "drive-web-second");
|
||||
let server_api = WorkspaceServerApi::new(config, store.clone());
|
||||
let server_api = WorkspaceServerApi::new_for_test(config, store.clone());
|
||||
let server = LoopbackServer::start(workspace_server_router(server_api.clone())).await;
|
||||
// Exercise T-717's real repositoryless Workspace bootstrap, not a
|
||||
// pre-existing test Workspace with its repository merely hidden in UI.
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
use super::*;
|
||||
use crate::db_executor::{
|
||||
DbExecutor, DbExecutorLimits, DbOperationLifetime, with_operation_lifetime,
|
||||
};
|
||||
use std::{
|
||||
num::NonZeroUsize,
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
mpsc,
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
fn owner() -> DbExecutor {
|
||||
DbExecutor::new(DbExecutorLimits {
|
||||
workers: NonZeroUsize::new(1).unwrap(),
|
||||
queued: NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
async fn observed<T>(future: impl std::future::Future<Output = T>) -> T {
|
||||
tokio::time::timeout(Duration::from_secs(2), future)
|
||||
.await
|
||||
.expect("owned protocol stage did not progress")
|
||||
}
|
||||
struct Release(Option<mpsc::Sender<()>>);
|
||||
impl Release {
|
||||
fn release(&mut self) {
|
||||
if let Some(tx) = self.0.take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Drop for Release {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
}
|
||||
}
|
||||
struct Lifetime(Arc<AtomicUsize>);
|
||||
struct Retained(Arc<AtomicUsize>);
|
||||
impl Drop for Retained {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
impl DbOperationLifetime for Lifetime {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
Box::new(Retained(self.0.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_embedded_endpoint_capture_keeps_actual_host_budget_off_tokio() {
|
||||
let (runtime, worker, execution) = runtime_with_worker_backend();
|
||||
let (started, held) = tokio::sync::oneshot::channel();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let mut release = Release(Some(tx));
|
||||
let thread = std::thread::spawn(move || {
|
||||
let _guard = execution.contexts.lock().unwrap();
|
||||
let _ = started.send(());
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5));
|
||||
});
|
||||
observed(held).await.unwrap();
|
||||
let executor = owner();
|
||||
let leases = Arc::new(AtomicUsize::new(0));
|
||||
let mut capture = Box::pin(with_operation_lifetime(
|
||||
Arc::new(Lifetime(leases.clone())),
|
||||
connect_embedded_worker_protocol(
|
||||
embedded_protocol_test_source(&runtime, &worker),
|
||||
embedded_protocol_test_scope(),
|
||||
executor.clone(),
|
||||
),
|
||||
));
|
||||
assert!(
|
||||
futures::poll!(&mut capture).is_pending(),
|
||||
"endpoint Runtime lock must not run on Tokio"
|
||||
);
|
||||
observed(tokio::time::sleep(Duration::from_millis(1))).await;
|
||||
assert_eq!(
|
||||
observed(tokio::spawn(async { "control" })).await.unwrap(),
|
||||
"control"
|
||||
);
|
||||
drop(capture);
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 1);
|
||||
executor.dispatch(|| {}).unwrap(); // only the one queued slot is left
|
||||
let error = executor.run(|| Ok::<_, Error>(())).await.unwrap_err();
|
||||
assert!(matches!(error, Error::DbCapacityExceeded));
|
||||
release.release();
|
||||
thread.join().unwrap();
|
||||
observed(executor.drain()).await.unwrap();
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_embedded_shutdown_commits_fs_receipt_after_the_pump_disappears() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let execution = Arc::new(DeterministicExecutionBackend::default());
|
||||
let options = worker_runtime::fs_store::FsRuntimeStoreOptions {
|
||||
root: dir.path().join("runtime"),
|
||||
runtime_id: "owned-protocol".into(),
|
||||
display_name: None,
|
||||
};
|
||||
let runtime = worker_runtime::Runtime::with_fs_store_and_execution_backend(
|
||||
options.clone(),
|
||||
execution.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(runtime_test_bundle()).unwrap();
|
||||
let worker = runtime
|
||||
.create_worker_scoped(&embedded_protocol_test_scope(), runtime_create_request())
|
||||
.unwrap()
|
||||
.worker_ref;
|
||||
let executor = owner();
|
||||
let mut connection = connect_embedded_worker_protocol(
|
||||
embedded_protocol_test_source(&runtime, &worker),
|
||||
embedded_protocol_test_scope(),
|
||||
executor.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
observed(connection.events.recv()).await.unwrap(),
|
||||
protocol::Event::Snapshot { .. }
|
||||
));
|
||||
let (entered, held) = tokio::sync::oneshot::channel();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let mut release = Release(Some(tx));
|
||||
let endpoint = execution
|
||||
.contexts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&worker)
|
||||
.unwrap()
|
||||
.clone();
|
||||
let gate_finished = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let finished = gate_finished.clone();
|
||||
*endpoint.dispatch_hook.lock().unwrap() = Some(Box::new(move || {
|
||||
let _ = entered.send(());
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5));
|
||||
finished.store(true, Ordering::SeqCst);
|
||||
}));
|
||||
connection
|
||||
.methods
|
||||
.send(protocol::Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(700),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
observed(held).await.unwrap(); // actual Runtime method, after its durable stop admission
|
||||
observed(tokio::time::sleep(Duration::from_millis(1))).await;
|
||||
assert_eq!(
|
||||
observed(tokio::spawn(async { "control" })).await.unwrap(),
|
||||
"control"
|
||||
);
|
||||
assert!(
|
||||
!gate_finished.load(Ordering::SeqCst),
|
||||
"watchdog bailout is not Tokio progress"
|
||||
);
|
||||
connection
|
||||
.termination
|
||||
.terminate(WorkerProtocolEnd::Disconnected);
|
||||
drop(connection);
|
||||
executor.dispatch(|| {}).unwrap();
|
||||
assert!(matches!(
|
||||
executor.run(|| Ok::<_, Error>(())).await.unwrap_err(),
|
||||
Error::DbCapacityExceeded
|
||||
));
|
||||
release.release();
|
||||
observed(executor.drain()).await.unwrap();
|
||||
assert_eq!(
|
||||
runtime.worker_detail(&worker).unwrap().status,
|
||||
worker_runtime::catalog::WorkerStatus::Stopped
|
||||
);
|
||||
// Read the actual fs-store aggregate written by the accepted method. The
|
||||
// deterministic execution fixture has no restorable Profile archive, so
|
||||
// restarting it would test archive recovery rather than this commit owner.
|
||||
let record: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(
|
||||
options
|
||||
.root
|
||||
.join("workers")
|
||||
.join(worker.worker_id.to_string())
|
||||
.join("worker.json"),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
record["status"], "stopped",
|
||||
"cancelled pump lost its fs-store commit: {record}"
|
||||
);
|
||||
assert_ne!(
|
||||
record["execution_state"]["state"],
|
||||
"reconciliation_required"
|
||||
);
|
||||
}
|
||||
@@ -11,6 +11,7 @@ async fn embedded_event_overflow_terminates_without_losing_the_admitted_prefix()
|
||||
let mut connection = connect_embedded_worker_protocol(
|
||||
embedded_protocol_test_source(&runtime, &worker_ref),
|
||||
embedded_protocol_test_scope(),
|
||||
HostExecutors::default().host().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -69,6 +70,7 @@ async fn embedded_protocol_delivers_snapshot_then_ordered_replies_and_live_event
|
||||
let mut connection = connect_embedded_worker_protocol(
|
||||
embedded_protocol_test_source(&runtime, &worker_ref),
|
||||
embedded_protocol_test_scope(),
|
||||
HostExecutors::default().host().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -105,9 +107,13 @@ async fn embedded_protocol_delivers_snapshot_then_ordered_replies_and_live_event
|
||||
async fn embedded_overflow_requires_explicit_reconnect_and_stop_ends_the_new_connection() {
|
||||
let (runtime, worker_ref, execution) = runtime_with_worker_backend();
|
||||
let source = embedded_protocol_test_source(&runtime, &worker_ref);
|
||||
let mut old = connect_embedded_worker_protocol(source.clone(), embedded_protocol_test_scope())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut old = connect_embedded_worker_protocol(
|
||||
source.clone(),
|
||||
embedded_protocol_test_scope(),
|
||||
HostExecutors::default().host().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_embedded_protocol_snapshot(next_embedded_protocol_event(&mut old).await, &worker_ref);
|
||||
tokio::time::timeout(PROTOCOL_TEST_TIMEOUT, async {
|
||||
for index in 0..=WORKER_PROTOCOL_EVENT_CAPACITY {
|
||||
@@ -127,9 +133,13 @@ async fn embedded_overflow_requires_explicit_reconnect_and_stop_ends_the_new_con
|
||||
|
||||
// Overflow ends only the attached connection, not its Worker. Reconnection
|
||||
// is explicit and starts from a fresh execution snapshot, not old backlog.
|
||||
let mut fresh = connect_embedded_worker_protocol(source, embedded_protocol_test_scope())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut fresh = connect_embedded_worker_protocol(
|
||||
source,
|
||||
embedded_protocol_test_scope(),
|
||||
HostExecutors::default().host().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_embedded_protocol_snapshot(next_embedded_protocol_event(&mut fresh).await, &worker_ref);
|
||||
fresh
|
||||
.methods
|
||||
@@ -767,6 +777,9 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) {
|
||||
std::time::Duration::from_secs(if explicit_termination { 10 } else { 20 })
|
||||
);
|
||||
assert!(execution.protocol_methods().is_empty());
|
||||
// SQLite jobs run on external threads; paused Tokio time cannot observe their
|
||||
// progress and may auto-advance a reconnect timeout before SQL completes.
|
||||
tokio::time::resume();
|
||||
assert_same_worker_reconnect_and_stop(&worker, &context).await;
|
||||
}
|
||||
|
||||
@@ -794,6 +807,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub
|
||||
let mut connection = connect_embedded_worker_protocol(
|
||||
embedded_protocol_test_source(&runtime, &worker_ref),
|
||||
embedded_protocol_test_scope(),
|
||||
HostExecutors::default().host().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -844,7 +858,7 @@ async fn embedded_disconnect_and_last_owner_drop_release_the_execution_event_sub
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[tokio::test]
|
||||
async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_same_worker_progress()
|
||||
{
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -1,34 +1,45 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_session_callback_completes_while_create_and_deletion_are_waiting() {
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn subject_callbacks_finish_before_waiting_host_parents_release_capacity() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let mut api = test_api(workspace.path()).await;
|
||||
let identity = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
|
||||
configure_runtime_request_auth(&mut api, &identity, "runtime-test");
|
||||
let subjektiv = open_subjektiv_store(&api).unwrap();
|
||||
let subject = subjektiv
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("companion").unwrap())
|
||||
.unwrap();
|
||||
let singleton_key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
|
||||
let memory = api
|
||||
.config_store
|
||||
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
|
||||
.unwrap();
|
||||
let reserved = api
|
||||
.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
"runtime-test",
|
||||
"subject-start",
|
||||
&"f".repeat(64),
|
||||
Some(&singleton_key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
let worker_id = reserved.worker_id.to_string();
|
||||
let server = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
|
||||
// Reuse the fixture API, but exercise the real outer dispatcher and both auth layers.
|
||||
let mut callbacks = Vec::new();
|
||||
for index in 0..2 {
|
||||
let subject = subjektiv
|
||||
.create_subject(
|
||||
crate::subjektiv::SubjectRole::new(format!("companion-{index}")).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let singleton_key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
|
||||
let reserved = api
|
||||
.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
"runtime-test",
|
||||
&format!("subject-start-{index}"),
|
||||
&"f".repeat(64),
|
||||
Some(&singleton_key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
callbacks.push((
|
||||
subject.id,
|
||||
reserved.worker_id.to_string(),
|
||||
session_store::new_session_id().to_string(),
|
||||
));
|
||||
}
|
||||
let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
|
||||
// Real dispatcher, source proof authentication, live singleton authority and
|
||||
// attribution SQL. Only Runtime's create/restore wait is modeled below; no
|
||||
// product process or complete create endpoint is spawned by this route test.
|
||||
server
|
||||
.apis
|
||||
.lock()
|
||||
@@ -39,61 +50,210 @@ async fn subject_session_callback_completes_while_create_and_deletion_are_waitin
|
||||
build_inner_router(api.clone()),
|
||||
);
|
||||
let gate = server.admission_for_workspace(TEST_WORKSPACE_ID).await;
|
||||
let create = gate.admit(false).unwrap();
|
||||
let mut deletion = Box::pin(gate.lock_deletion());
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
let host = api.host_executors.host().unwrap(); // production 2 workers/16 queued
|
||||
let mut parents = Vec::new();
|
||||
let mut finish_callbacks = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let create = gate.admit(false).unwrap();
|
||||
let (entered, ready) = tokio::sync::oneshot::channel();
|
||||
let (finish_callback, callback_reply) = std::sync::mpsc::channel();
|
||||
finish_callbacks.push(finish_callback);
|
||||
parents.push(tokio::spawn(run_host_workflow(host.clone(), async move {
|
||||
let _create = create;
|
||||
entered.send(()).unwrap();
|
||||
// The actual Host parent cannot finish until its mandatory callbacks
|
||||
// return. Timeout/disconnection is failure cleanup, never success.
|
||||
callback_reply
|
||||
.recv_timeout(std::time::Duration::from_secs(5))
|
||||
.map_err(|error| {
|
||||
Error::InvalidInput(format!("Runtime callback did not complete: {error}"))
|
||||
})?;
|
||||
Ok::<_, Error>(())
|
||||
})));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), ready)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
let app = workspace_server_router(server);
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions");
|
||||
let session_id = session_store::new_session_id().to_string();
|
||||
let body = serde_json::to_vec(&json!({ "session_id": session_id, "create_if_missing": true }))
|
||||
.unwrap();
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
app.clone().oneshot(runtime_source_request(
|
||||
&identity,
|
||||
Some(&worker_id),
|
||||
"POST",
|
||||
&path,
|
||||
body.clone(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("Session attribution must not wait for its own Worker creation")
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{response_body:?}");
|
||||
let response: Value = serde_json::from_slice(&response_body).unwrap();
|
||||
assert_eq!(response["subject_id"], subject.id);
|
||||
assert_eq!(response["session_id"], session_id);
|
||||
let stored = subjektiv.session_attribution(&session_id).unwrap().unwrap();
|
||||
assert_eq!(stored.worker_id, worker_id);
|
||||
assert_eq!(stored.subject_id, subject.id);
|
||||
|
||||
// The callback lane does not bypass authentication.
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
app.clone().oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(&path)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(body))
|
||||
let memory_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/memory");
|
||||
for (_, worker_id, _) in &callbacks {
|
||||
// Resident context is also a pure Subject callback used by Runtime
|
||||
// bootstrap/restore. It must not reenter the waiting Host lane either.
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
app.clone().oneshot(runtime_source_request(
|
||||
&identity,
|
||||
Some(worker_id),
|
||||
"POST",
|
||||
&memory_path,
|
||||
serde_json::to_vec(&server_api::SubjektivMemoryBackendRequest {
|
||||
operation: server_api::SubjektivMemoryBackendOperation::ResidentContext(
|
||||
Default::default(),
|
||||
),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("resident context must not wait for its Host parents")
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{body:?}");
|
||||
let resident: server_api::SubjektivMemoryBackendResponse =
|
||||
serde_json::from_slice(&body).unwrap();
|
||||
assert!(matches!(
|
||||
resident,
|
||||
server_api::SubjektivMemoryBackendResponse::ResidentContext(_)
|
||||
));
|
||||
}
|
||||
// Resident memory is not an allowed deletion-drain callback. Preserve that
|
||||
// admission fence; only the exact Session attribution route reenters below.
|
||||
let mut deletion = Box::pin(gate.lock_deletion());
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
drop(create);
|
||||
for (subject_id, worker_id, session_id) in &callbacks {
|
||||
let body =
|
||||
serde_json::to_vec(&json!({ "session_id": session_id, "create_if_missing": true }))
|
||||
.unwrap();
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
app.clone().oneshot(runtime_source_request(
|
||||
&identity,
|
||||
Some(worker_id),
|
||||
"POST",
|
||||
&path,
|
||||
body,
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("Session attribution must not wait for its Host parents")
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{response_body:?}");
|
||||
let response: Value = serde_json::from_slice(&response_body).unwrap();
|
||||
assert_eq!(response["subject_id"], *subject_id);
|
||||
assert_eq!(response["session_id"], *session_id);
|
||||
let stored = subjektiv.session_attribution(session_id).unwrap().unwrap();
|
||||
assert_eq!(stored.worker_id, *worker_id);
|
||||
assert_eq!(stored.subject_id, *subject_id);
|
||||
}
|
||||
// Neither actual parent nor its admission lease was released to make these
|
||||
// callbacks succeed. Only durable attribution completion releases them.
|
||||
assert!(parents.iter().all(|parent| !parent.is_finished()));
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
for finish_callback in finish_callbacks {
|
||||
finish_callback.send(()).unwrap();
|
||||
}
|
||||
for parent in parents {
|
||||
parent.await.unwrap().unwrap();
|
||||
}
|
||||
host.drain().await.unwrap();
|
||||
api.store.db_executor().drain().await.unwrap();
|
||||
let _deletion = tokio::time::timeout(std::time::Duration::from_secs(2), deletion)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(gate.admit(true).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn consolidation_subject_callbacks_are_denied_without_runtime_lookup() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let mut api = test_api(workspace.path()).await;
|
||||
let identity = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
|
||||
configure_runtime_request_auth(&mut api, &identity, "runtime-test");
|
||||
let subjektiv = open_subjektiv_store(&api).unwrap();
|
||||
let subject = subjektiv
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("companion").unwrap())
|
||||
.unwrap();
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{}", subject.id);
|
||||
let memory = api
|
||||
.config_store
|
||||
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
|
||||
.unwrap();
|
||||
let reservation = api
|
||||
.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
"runtime-test",
|
||||
"consolidation-start",
|
||||
&"f".repeat(64),
|
||||
Some(&key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
// runtime-test has no Worker adapter: authority denial must not try to look
|
||||
// up its profile/tags through Runtime before rejecting this disallowed role.
|
||||
let server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
|
||||
server
|
||||
.apis
|
||||
.lock()
|
||||
.await
|
||||
.insert(TEST_WORKSPACE_ID.into(), api.clone());
|
||||
server
|
||||
.routers
|
||||
.lock()
|
||||
.await
|
||||
.insert(TEST_WORKSPACE_ID.into(), build_inner_router(api.clone()));
|
||||
let app = workspace_server_router(server);
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions");
|
||||
let worker_id = reservation.worker_id.to_string();
|
||||
let session_id = session_store::new_session_id().to_string();
|
||||
for (callback_path, callback_body) in [
|
||||
(
|
||||
path.clone(),
|
||||
serde_json::to_vec(&json!({"session_id": session_id, "create_if_missing": true}))
|
||||
.unwrap(),
|
||||
),
|
||||
(
|
||||
format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/memory"),
|
||||
serde_json::to_vec(&server_api::SubjektivMemoryBackendRequest {
|
||||
operation: server_api::SubjektivMemoryBackendOperation::ResidentContext(
|
||||
Default::default(),
|
||||
),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
] {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(runtime_source_request(
|
||||
&identity,
|
||||
Some(&worker_id),
|
||||
"POST",
|
||||
&callback_path,
|
||||
callback_body,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{callback_path}: {body:?}");
|
||||
}
|
||||
assert!(
|
||||
subjektiv
|
||||
.session_attribution(&session_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(path)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(
|
||||
json!({"session_id": session_id, "create_if_missing": true}).to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workspace_deletion_waits_for_session_attribution_and_fences_both_lanes() {
|
||||
let gate = Arc::new(WorkspaceAdmission::default());
|
||||
|
||||
@@ -144,7 +144,7 @@ impl LaunchFixture {
|
||||
async fn new(remote: bool) -> Self {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let config = test_server_config(temp.path());
|
||||
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap();
|
||||
let backend = Arc::new(LaunchExecutionBackend::default());
|
||||
let api =
|
||||
WorkspaceApi::new_with_execution_backend(config, Arc::new(store), backend.clone())
|
||||
|
||||
@@ -151,7 +151,7 @@ impl Fixture {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(temp.path());
|
||||
config.repositories.clear();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
let mut api = WorkspaceApi::new_with_execution_backend(
|
||||
config,
|
||||
store,
|
||||
|
||||
@@ -830,7 +830,11 @@ async fn grant_replacement_while_admission_waits_cannot_use_another_grants_lock(
|
||||
};
|
||||
let admission = handle.set_pinned(&context, true);
|
||||
tokio::pin!(admission);
|
||||
// Poll deterministically to the held lock before replacing grant authority.
|
||||
// The first poll can now be waiting for the offloaded authority read, not
|
||||
// the held grant lock. Drain this fixture's actual SQL, then poll its saved
|
||||
// reply to reach admission before replacing authority.
|
||||
assert!(futures::poll!(admission.as_mut()).is_pending());
|
||||
api.store.db_executor().drain().await.unwrap();
|
||||
assert!(futures::poll!(admission.as_mut()).is_pending());
|
||||
api.store
|
||||
.revoke_worker_control_grant(
|
||||
@@ -1329,8 +1333,9 @@ async fn restore_guard_internal_stale_conflict_retires_intent_for_later_explicit
|
||||
);
|
||||
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||
// Retirement, unlike in-memory refresh, must survive caller journal restart.
|
||||
api.config_store =
|
||||
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
|
||||
api.config_store = Arc::new(
|
||||
SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap(),
|
||||
);
|
||||
let fresh = api.runtime.worker(&identity).unwrap();
|
||||
let next = api
|
||||
.config_store
|
||||
@@ -1490,7 +1495,7 @@ async fn restore_guard_internal_intent_survives_client_store_restart_without_rep
|
||||
);
|
||||
// Reopen the durable caller journal, discarding all client-side memory of request identity.
|
||||
api.config_store =
|
||||
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(api.config.database_path.clone()).unwrap());
|
||||
let pinned = api
|
||||
.config_store
|
||||
.pin_internal_worker_restore_intent(
|
||||
@@ -1599,7 +1604,7 @@ async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_w
|
||||
crate::repository_source::repository_source_fingerprint(&source);
|
||||
config.repositories[0].source = source;
|
||||
config.repositories[0].path = None;
|
||||
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap();
|
||||
let execution = Arc::new(DeterministicExecutionBackend::default());
|
||||
execution.accept_restores();
|
||||
let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone())
|
||||
|
||||
@@ -56,7 +56,7 @@ impl Fixture {
|
||||
api.runtime.register_or_replace(runtime);
|
||||
let now = Arc::new(AtomicI64::new(START));
|
||||
let clock = now.clone();
|
||||
let mut server = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
|
||||
let mut server = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
|
||||
server.proof_clock = Arc::new(move || clock.load(Ordering::SeqCst));
|
||||
server
|
||||
.apis
|
||||
@@ -330,9 +330,38 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
|
||||
let f = Fixture::new().await;
|
||||
let mut a = f.attach("1");
|
||||
let (first, reply) = f.start_read("1", &mut a).await;
|
||||
let mut second = Box::pin(f.app.clone().oneshot(f.operation("1")));
|
||||
// Poll through dispatch/auth until the handler waits on its resource lock.
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let request = f.operation("1");
|
||||
let proof = request
|
||||
.headers()
|
||||
.get(worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER)
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap();
|
||||
let jti = worker_runtime::auth::decode_runtime_request_source_claims(proof)
|
||||
.unwrap()
|
||||
.jti;
|
||||
let mut second = Box::pin(f.app.clone().oneshot(request));
|
||||
// DB auth is now genuinely asynchronous. Observe durable proof consumption,
|
||||
// not a guessed number of polls, before advancing the injected proof clock.
|
||||
let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
tokio::time::timeout(BOUND, async {
|
||||
loop {
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let consumed: bool = connection
|
||||
.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM worker_mutation_source_proof_jtis WHERE jti=?1)",
|
||||
[&jti],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
if consumed {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second request never authenticated before resource wait");
|
||||
assert!(
|
||||
a.try_recv().is_err(),
|
||||
"same session read escaped resource exclusion"
|
||||
@@ -342,7 +371,12 @@ async fn same_worker_session_stays_exclusive_but_proof_is_verified_before_waitin
|
||||
assert_json(first.await.unwrap(), StatusCode::OK).await;
|
||||
assert!(futures::poll!(&mut second).is_pending());
|
||||
let Some(ExternalProviderCommand::Operation { response, .. }) =
|
||||
tokio::time::timeout(BOUND, a.recv()).await.unwrap()
|
||||
tokio::time::timeout(BOUND, async {
|
||||
tokio::select! {
|
||||
command = a.recv() => command,
|
||||
result = second.as_mut() => panic!("second read ended before provider dispatch: {:?}", result.unwrap().status()),
|
||||
}
|
||||
}).await.unwrap()
|
||||
else {
|
||||
panic!("expected second read")
|
||||
};
|
||||
@@ -848,12 +882,18 @@ async fn create_and_restore_session_callbacks_finish_while_deletion_start_waits(
|
||||
}),
|
||||
);
|
||||
let mut deletion = Box::pin(f.app.clone().oneshot(request));
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await;
|
||||
assert!(
|
||||
gate.admit(false).is_none(),
|
||||
"deletion start did not fence admission"
|
||||
);
|
||||
tokio::time::timeout(BOUND, async {
|
||||
loop {
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
if gate.admit(false).is_none() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("deletion start did not fence admission after async authentication");
|
||||
let session_id = session_store::new_session_id().to_string();
|
||||
let callback_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions");
|
||||
let callback = f.request(
|
||||
|
||||
@@ -57,6 +57,19 @@ impl WorkerOperationContext {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) async fn from_request_async(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
) -> std::result::Result<Self, server_api::RepositoryApiError> {
|
||||
let owned_api = api.clone();
|
||||
let context = context.clone();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || Ok::<_, Error>(Self::from_request(&owned_api, &context)))
|
||||
.await
|
||||
.map_err(|error| ApiError::from(error).into_repository_api_error())?
|
||||
}
|
||||
|
||||
pub(super) fn from_request(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
@@ -162,8 +175,20 @@ impl<Services> WorkspaceWorker<Services> {
|
||||
source: crate::worker_source::VerifiedWorkerMutationSource,
|
||||
reason: &str,
|
||||
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
||||
self.current_record()
|
||||
.map_err(|error| error.error.to_string())?;
|
||||
let removal = self.removal.clone();
|
||||
let identity = self.identity.clone();
|
||||
self.removal
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
removal
|
||||
.store
|
||||
.get_worker_registry(&removal.workspace_id, &identity)?
|
||||
.ok_or_else(|| Error::UnknownWorker { worker: identity })
|
||||
.map(drop)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
self.removal
|
||||
.execute_async(
|
||||
source,
|
||||
@@ -225,6 +250,20 @@ impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWor
|
||||
}
|
||||
|
||||
impl WorkspaceWorker {
|
||||
pub(crate) async fn resolve_async(
|
||||
api: &WorkspaceApi,
|
||||
runtime_id: &str,
|
||||
reference: &str,
|
||||
) -> Result<Self> {
|
||||
let owned_api = api.clone();
|
||||
let runtime_id = runtime_id.to_owned();
|
||||
let reference = reference.to_owned();
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || Self::resolve(&owned_api, &runtime_id, &reference))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result<Self> {
|
||||
Self::resolve_with_services(
|
||||
api.clone(),
|
||||
@@ -239,7 +278,36 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
permission: &str,
|
||||
) -> ApiResult<Option<tokio::sync::OwnedMutexGuard<()>>> {
|
||||
self.current_record()?;
|
||||
let worker = self.clone();
|
||||
let initial_context = context.clone();
|
||||
let permission = permission.to_owned();
|
||||
let initial_permission = permission.clone();
|
||||
let grant = self
|
||||
.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<Option<WorkerControlGrantRecord>> {
|
||||
worker.current_record()?;
|
||||
let WorkerOperationContext::WorkerControl { controller } = initial_context else {
|
||||
return Ok(None);
|
||||
};
|
||||
if worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(worker.api.workspace_id(), &controller)?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::UnknownWorker { worker: controller }.into());
|
||||
}
|
||||
Ok(Some(authorize_known_worker_permission(
|
||||
&worker.api,
|
||||
worker.api.workspace_id(),
|
||||
&controller,
|
||||
&worker.identity,
|
||||
&initial_permission,
|
||||
)?))
|
||||
})
|
||||
.await?;
|
||||
match context {
|
||||
#[cfg(test)]
|
||||
WorkerOperationContext::Backend => Ok(None),
|
||||
@@ -248,42 +316,35 @@ impl WorkspaceWorker {
|
||||
Ok(None)
|
||||
}
|
||||
WorkerOperationContext::WorkerControl { controller } => {
|
||||
if self
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(self.api.workspace_id(), controller)?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::UnknownWorker {
|
||||
worker: controller.clone(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let grant = authorize_known_worker_permission(
|
||||
&self.api,
|
||||
self.api.workspace_id(),
|
||||
controller,
|
||||
&self.identity,
|
||||
permission,
|
||||
)?;
|
||||
let grant = grant.expect("Worker control context resolved a grant");
|
||||
// Await admission outside the DB lane; the recheck owns the guard
|
||||
// until it finishes, even if its async waiter is cancelled.
|
||||
let guard = worker_control_lock(&self.api, &grant.grant_id)
|
||||
.lock_owned()
|
||||
.await;
|
||||
self.current_record()?;
|
||||
let current_grant = authorize_known_worker_permission(
|
||||
&self.api,
|
||||
self.api.workspace_id(),
|
||||
controller,
|
||||
&self.identity,
|
||||
permission,
|
||||
)?;
|
||||
if current_grant.grant_id != grant.grant_id {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Worker control grant changed while waiting for admission".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(Some(guard))
|
||||
let worker = self.clone();
|
||||
let controller = controller.clone();
|
||||
self.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<_> {
|
||||
worker.current_record()?;
|
||||
let current_grant = authorize_known_worker_permission(
|
||||
&worker.api,
|
||||
worker.api.workspace_id(),
|
||||
&controller,
|
||||
&worker.identity,
|
||||
&permission,
|
||||
)?;
|
||||
if current_grant.grant_id != grant.grant_id {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Worker control grant changed while waiting for admission".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(Some(guard))
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -293,8 +354,16 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
) -> ApiResult<InternalWorkerRestoreResult> {
|
||||
let _guard = self.authorize_operation(context, "restore").await?;
|
||||
self.restore_internal(request)
|
||||
let guard = self.authorize_operation(context, "restore").await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
worker.restore_internal(request)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Trusted Backend callers still supply an explicit observed intent. Runtime
|
||||
@@ -312,27 +381,40 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: WorkerLifecycleRequest,
|
||||
) -> ApiResult<WorkerLifecycleResult> {
|
||||
let _guard = self.authorize_operation(context, "stop").await?;
|
||||
let result = self
|
||||
.api
|
||||
.runtime
|
||||
.stop_worker(&self.identity, request)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if result.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(result);
|
||||
}
|
||||
parse_runtime_worker_id_for_registry(&self.identity.worker_id)?;
|
||||
let session_lock = current_worker_session_lock(&self.api, &self.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
close_current_worker_session_locked(&self.api, &self.identity).await?;
|
||||
if let Some(record) = self
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(self.api.workspace_id(), &self.identity)?
|
||||
{
|
||||
sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?;
|
||||
}
|
||||
Ok(result)
|
||||
let guard = self.authorize_operation(context, "stop").await?;
|
||||
let worker = self.clone();
|
||||
// Once admitted, Runtime stop, session close and linked Workdir
|
||||
// publication have one owner. Never hand an accepted stop's guards
|
||||
// back to its cancellable waiter or acquire this Host lane again.
|
||||
run_host_workflow(self.api.host_executors.host()?, async move {
|
||||
let _control = guard;
|
||||
let result = worker
|
||||
.api
|
||||
.runtime
|
||||
.stop_worker(&worker.identity, request)
|
||||
.map_err(|error| error.into_error())?;
|
||||
if result.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(result);
|
||||
}
|
||||
parse_runtime_worker_id_for_registry(&worker.identity.worker_id)?;
|
||||
let _session = current_worker_session_lock(&worker.api, &worker.identity)
|
||||
.lock_owned()
|
||||
.await;
|
||||
close_current_worker_session_locked(&worker.api, &worker.identity).await?;
|
||||
if let Some(record) = worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(worker.api.workspace_id(), &worker.identity)?
|
||||
{
|
||||
sync_linked_workdir_after_worker_stop(
|
||||
&worker.api,
|
||||
&worker.identity.runtime_id,
|
||||
&record,
|
||||
)?;
|
||||
}
|
||||
Ok(result)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn cancel(
|
||||
@@ -340,12 +422,20 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
request: WorkerLifecycleRequest,
|
||||
) -> ApiResult<WorkerLifecycleResult> {
|
||||
let _guard = self.authorize_operation(context, "cancel").await?;
|
||||
Ok(self
|
||||
.api
|
||||
.runtime
|
||||
.cancel_worker(&self.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
let guard = self.authorize_operation(context, "cancel").await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let _guard = guard;
|
||||
Ok(worker
|
||||
.api
|
||||
.runtime
|
||||
.cancel_worker(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn input(
|
||||
@@ -358,46 +448,61 @@ impl WorkspaceWorker {
|
||||
} else {
|
||||
"send_input"
|
||||
};
|
||||
let _guard = self.authorize_operation(context, permission).await?;
|
||||
let guard = self.authorize_operation(context, permission).await?;
|
||||
let session_lock = current_worker_session_lock(&self.api, &self.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
self.ensure_interactive()?;
|
||||
let session_guard = session_lock.lock_owned().await;
|
||||
self.ensure_interactive().await?;
|
||||
if matches!(context, WorkerOperationContext::WorkerControl { .. })
|
||||
&& request.kind == WorkerInputKind::User
|
||||
{
|
||||
request.kind = WorkerInputKind::UserIfIdle;
|
||||
}
|
||||
Ok(self
|
||||
.api
|
||||
.runtime
|
||||
.send_input(&self.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let (_control, _session) = (guard, session_guard);
|
||||
Ok(worker
|
||||
.api
|
||||
.runtime
|
||||
.send_input(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn ensure_interactive(&self) -> ApiResult<()> {
|
||||
if let Some(claim) = self
|
||||
.api
|
||||
async fn ensure_interactive(&self) -> ApiResult<()> {
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.store
|
||||
.get_pending_ticket_assignment_operation_for_worker(
|
||||
self.api.workspace_id(),
|
||||
&self.identity,
|
||||
)?
|
||||
{
|
||||
return Err(Error::TicketAssignmentConflict(format!(
|
||||
"Worker has pending Ticket claim {}; reconcile that exact operation before admitting input",
|
||||
claim.operation_id,
|
||||
)).into());
|
||||
}
|
||||
if self
|
||||
.api
|
||||
.store
|
||||
.worker_registry_projection(self.api.workspace_id(), &self.identity)?
|
||||
.and_then(|projection| projection.job)
|
||||
.is_some()
|
||||
{
|
||||
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
|
||||
}
|
||||
Ok(())
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<()> {
|
||||
if let Some(claim) = worker
|
||||
.api
|
||||
.store
|
||||
.get_pending_ticket_assignment_operation_for_worker(
|
||||
worker.api.workspace_id(),
|
||||
&worker.identity,
|
||||
)?
|
||||
{
|
||||
return Err(Error::TicketAssignmentConflict(format!(
|
||||
"Worker has pending Ticket claim {}; reconcile that exact operation before admitting input",
|
||||
claim.operation_id,
|
||||
)).into());
|
||||
}
|
||||
if worker
|
||||
.api
|
||||
.store
|
||||
.worker_registry_projection(worker.api.workspace_id(), &worker.identity)?
|
||||
.and_then(|projection| projection.job)
|
||||
.is_some()
|
||||
{
|
||||
return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into());
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn set_pinned(
|
||||
@@ -405,33 +510,44 @@ impl WorkspaceWorker {
|
||||
context: &WorkerOperationContext,
|
||||
pinned: bool,
|
||||
) -> ApiResult<WorkerRetentionResponse> {
|
||||
let _guard = self.authorize_operation(context, "pin").await?;
|
||||
let retention_state = if pinned { "pinned" } else { "normal" };
|
||||
let changed = self.api.worker_projection.publish_ordered(|| {
|
||||
self.api
|
||||
.store
|
||||
.update_worker_retention(
|
||||
self.api.workspace_id(),
|
||||
&self.identity,
|
||||
retention_state,
|
||||
now_registry_timestamp().as_str(),
|
||||
)
|
||||
.map(|commit| (!commit.changes.is_empty(), commit))
|
||||
})?;
|
||||
if !changed {
|
||||
return Err(cleanup_api_error(
|
||||
&self.identity.runtime_id,
|
||||
"workspace_worker_retention_unknown_worker",
|
||||
"Worker is not known to the Backend registry",
|
||||
));
|
||||
}
|
||||
Ok(WorkerRetentionResponse {
|
||||
workspace_id: self.api.workspace_id().to_owned(),
|
||||
runtime_id: self.identity.runtime_id.clone(),
|
||||
worker_id: self.identity.worker_id.clone(),
|
||||
pinned,
|
||||
retention_state: retention_state.into(),
|
||||
})
|
||||
let guard = self.authorize_operation(context, "pin").await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || -> ApiResult<WorkerRetentionResponse> {
|
||||
// Publication and its DB mutation are one owned stage. Accepted
|
||||
// work retains control admission through commit and publication.
|
||||
let _guard = guard;
|
||||
let retention_state = if pinned { "pinned" } else { "normal" };
|
||||
let changed = worker.api.worker_projection.publish_ordered(|| {
|
||||
worker
|
||||
.api
|
||||
.store
|
||||
.update_worker_retention(
|
||||
worker.api.workspace_id(),
|
||||
&worker.identity,
|
||||
retention_state,
|
||||
now_registry_timestamp().as_str(),
|
||||
)
|
||||
.map(|commit| (!commit.changes.is_empty(), commit))
|
||||
})?;
|
||||
if !changed {
|
||||
return Err(cleanup_api_error(
|
||||
&worker.identity.runtime_id,
|
||||
"workspace_worker_retention_unknown_worker",
|
||||
"Worker is not known to the Backend registry",
|
||||
));
|
||||
}
|
||||
Ok(WorkerRetentionResponse {
|
||||
workspace_id: worker.api.workspace_id().to_owned(),
|
||||
runtime_id: worker.identity.runtime_id.clone(),
|
||||
worker_id: worker.identity.worker_id.clone(),
|
||||
pinned,
|
||||
retention_state: retention_state.into(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// The caller owns the full cleanup-plan digest check. The removal service
|
||||
@@ -611,7 +727,7 @@ impl WorkspaceWorkerMethodSender {
|
||||
.await?;
|
||||
let session_lock = current_worker_session_lock(&self.worker.api, &self.worker.identity);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
self.worker.ensure_interactive()?;
|
||||
self.worker.ensure_interactive().await?;
|
||||
let WorkerOperationContext::Browser { source, .. } = context else {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"Browser Worker protocol context required".into(),
|
||||
|
||||
@@ -72,6 +72,28 @@ pub(super) struct RequestLease {
|
||||
callback: bool,
|
||||
}
|
||||
|
||||
impl crate::db_executor::DbOperationLifetime for RequestLease {
|
||||
fn retain(&self) -> Box<dyn Send> {
|
||||
// This is continuation of an already admitted request, not fresh admission.
|
||||
// Retain its drain responsibility before transferring work to the DB owner;
|
||||
// dropping/aborting the HTTP waiter must not allow deletion past live SQL.
|
||||
let mut state = self
|
||||
.gate
|
||||
.state
|
||||
.lock()
|
||||
.expect("Workspace admission poisoned");
|
||||
if self.callback {
|
||||
state.callbacks += 1;
|
||||
} else {
|
||||
state.requests += 1;
|
||||
}
|
||||
Box::new(RequestLease {
|
||||
gate: self.gate.clone(),
|
||||
callback: self.callback,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RequestLease {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self
|
||||
|
||||
@@ -597,7 +597,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let config = test_server_config(dir.path());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -769,7 +769,7 @@ async fn delivery_real_fixture(
|
||||
DeliveryScriptedClient,
|
||||
) {
|
||||
let config = test_server_config(root);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1007,7 +1007,7 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
||||
tokio::task::yield_now().await;
|
||||
let reopened = WorkspaceApi::new_with_execution_backend(
|
||||
config.clone(),
|
||||
Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()),
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()),
|
||||
execution.clone(),
|
||||
)
|
||||
.await
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -51,14 +51,13 @@ async fn blocking<T: Send + 'static>(
|
||||
mutation: bool,
|
||||
action: impl FnOnce() -> workspace_drive::Result<T> + Send + 'static,
|
||||
) -> DriveResult<T> {
|
||||
tokio::task::spawn_blocking(action)
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || Ok::<_, Error>(action()))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
if mutation {
|
||||
DriveApiError::new(DriveApiErrorCode::OutcomeUnknown)
|
||||
} else {
|
||||
failure(503)
|
||||
}
|
||||
.map_err(|error| match error {
|
||||
Error::DbCapacityExceeded | Error::DbExecutorUnavailable => failure(503),
|
||||
_ if mutation => DriveApiError::new(DriveApiErrorCode::OutcomeUnknown),
|
||||
_ => failure(503),
|
||||
})?
|
||||
.map_err(|error| domain_error(error, mutation))
|
||||
}
|
||||
|
||||
@@ -24,7 +24,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> DriveApiError {
|
||||
})
|
||||
}
|
||||
fn mutation_error(error: Error) -> DriveApiError {
|
||||
if matches!(error, Error::Sqlite(_) | Error::Store(_)) {
|
||||
if matches!(
|
||||
error,
|
||||
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
|
||||
) {
|
||||
fail(DriveApiErrorCode::OutcomeUnknown)
|
||||
} else {
|
||||
pre_error(error)
|
||||
@@ -61,8 +64,9 @@ pub(super) async fn authorized_drive(
|
||||
// incomplete, or inconsistent with its transport identity.
|
||||
let headers = current_worker_contract_headers(context)
|
||||
.map_err(|_| fail(DriveApiErrorCode::Denied))?;
|
||||
let worker = current_worker_identity(api, workspace_id, &headers)
|
||||
.map_err(|_| fail(DriveApiErrorCode::Denied))?;
|
||||
let worker = authenticate_worker_mutation_source_owned(api, workspace_id, &headers)
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
if context.worker_source.as_ref().is_some_and(|source| {
|
||||
source.runtime_id != worker.runtime_id || source.worker_id != worker.worker_id
|
||||
}) {
|
||||
@@ -97,23 +101,27 @@ pub(super) async fn authorized_drive(
|
||||
};
|
||||
// Reject missing/read-only grants before upload body reception. This check
|
||||
// is admission only: every later operation rechecks under its own lock.
|
||||
tokio::task::spawn_blocking(move || {
|
||||
drive.check_authorized(write)?;
|
||||
Ok::<_, workspace_drive::Error>(drive)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
.map_err(|error| {
|
||||
fail(match error {
|
||||
workspace_drive::Error::Denied => DriveApiErrorCode::Denied,
|
||||
workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => {
|
||||
DriveApiErrorCode::Conflict
|
||||
}
|
||||
workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound,
|
||||
workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid,
|
||||
_ => DriveApiErrorCode::StorageUnavailable,
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
drive.check_authorized(write)?;
|
||||
Ok::<_, workspace_drive::Error>(drive)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
.map_err(|error| {
|
||||
fail(match error {
|
||||
workspace_drive::Error::Denied => DriveApiErrorCode::Denied,
|
||||
workspace_drive::Error::Fenced | workspace_drive::Error::Conflict => {
|
||||
DriveApiErrorCode::Conflict
|
||||
}
|
||||
workspace_drive::Error::NotFound => DriveApiErrorCode::NotFound,
|
||||
workspace_drive::Error::Invalid(_) => DriveApiErrorCode::Invalid,
|
||||
_ => DriveApiErrorCode::StorageUnavailable,
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn create_grant(
|
||||
@@ -132,9 +140,14 @@ pub(super) async fn create_grant(
|
||||
}
|
||||
}
|
||||
let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id);
|
||||
let operation_api = api.clone();
|
||||
let summary = api
|
||||
.runtime
|
||||
.worker(&worker)
|
||||
.host_executors
|
||||
.host()
|
||||
.map_err(pre_error)?
|
||||
.run(move || Ok::<_, Error>(operation_api.runtime.worker(&worker)))
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
.map_err(|_| fail(DriveApiErrorCode::NotFound))?;
|
||||
if summary.workspace.workspace_id.as_deref() != Some(workspace_id) {
|
||||
return Err(fail(DriveApiErrorCode::Denied));
|
||||
@@ -152,9 +165,10 @@ pub(super) async fn create_grant(
|
||||
revoked_at: None,
|
||||
};
|
||||
let store = api.store.clone();
|
||||
tokio::task::spawn_blocking(move || store.create_workspace_drive_grant(&grant))
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || store.create_workspace_drive_grant(&grant))
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
|
||||
.map_err(mutation_error)
|
||||
}
|
||||
|
||||
@@ -175,13 +189,12 @@ pub(super) async fn revoke_grant(
|
||||
actor.account_id.clone(),
|
||||
);
|
||||
let store = api.store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
store.revoke_workspace_drive_grant(&workspace, &grant, &actor)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::OutcomeUnknown))?
|
||||
.map_err(mutation_error)?
|
||||
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || store.revoke_workspace_drive_grant(&workspace, &grant, &actor))
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
.ok_or_else(|| fail(DriveApiErrorCode::NotFound))
|
||||
}
|
||||
|
||||
fn validate_grant_id(id: &str) -> DriveResult<()> {
|
||||
@@ -212,12 +225,14 @@ pub(super) async fn list_grants(
|
||||
}
|
||||
let workspace = workspace_id.to_owned();
|
||||
let store = api.store.clone();
|
||||
let mut grants = tokio::task::spawn_blocking(move || {
|
||||
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| fail(DriveApiErrorCode::StorageUnavailable))?
|
||||
.map_err(pre_error)?;
|
||||
let mut grants = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store.list_workspace_drive_grants(&workspace, limit + 1, query.after.as_deref())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let more = grants.len() > limit;
|
||||
grants.truncate(limit);
|
||||
let next_after = if more {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//! Cross-lane composition regressions for T743's three new async DB callers.
|
||||
//! Domain success/CAS/replay/rollback is covered by assignment_tracking_tests;
|
||||
//! these exercise real generated handlers and their finite execution owners.
|
||||
use super::*;
|
||||
|
||||
struct Release(Option<mpsc::Sender<()>>);
|
||||
impl Drop for Release {
|
||||
fn drop(&mut self) {
|
||||
self.0.take();
|
||||
}
|
||||
}
|
||||
|
||||
async fn queued_caller_keeps_owner(
|
||||
method: &str,
|
||||
suffix: &str,
|
||||
body: serde_json::Value,
|
||||
host: bool,
|
||||
) {
|
||||
let fixture = Fixture::new().await;
|
||||
let executor = if host {
|
||||
fixture.host_executor.clone()
|
||||
} else {
|
||||
fixture.executor.clone()
|
||||
};
|
||||
let store = fixture.store.clone();
|
||||
let (entered, waiting) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
let mut release = Release(Some(release));
|
||||
let watchdog = Arc::new(AtomicBool::new(false));
|
||||
let expired = watchdog.clone();
|
||||
// Occupy the actual owner and connection deterministically, not via sleeps
|
||||
// or a request that could still be waiting in its authentication stage.
|
||||
executor
|
||||
.dispatch(move || {
|
||||
store
|
||||
.with_conn(|_| {
|
||||
let _ = entered.send(());
|
||||
if permit.recv_timeout(WATCHDOG).is_err() {
|
||||
expired.store(true, Ordering::SeqCst);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
})
|
||||
.unwrap();
|
||||
observed(waiting).await.unwrap();
|
||||
let path = format!("/api/w/{WORKSPACE}/tickets/T-missing/{suffix}");
|
||||
let request = || Fixture::request(method, &path, body.clone());
|
||||
let (lifetime, accepted) = admission_observation();
|
||||
let mut queued = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
lifetime,
|
||||
fixture.inner_router().oneshot(request()),
|
||||
));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
observed(accepted).await.unwrap();
|
||||
let control = tokio::spawn(async { "independent control" });
|
||||
assert_eq!(observed(control).await.unwrap(), "independent control");
|
||||
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
|
||||
assert!(
|
||||
!watchdog.load(Ordering::SeqCst),
|
||||
"synchronous caller stalled Tokio until the watchdog"
|
||||
);
|
||||
drop(queued);
|
||||
// Cancellation of the waiter cannot free the accepted queued SQL slot.
|
||||
let rejected = observed(fixture.inner_router().oneshot(request()))
|
||||
.await
|
||||
.unwrap();
|
||||
let error = response_json(rejected, StatusCode::SERVICE_UNAVAILABLE).await;
|
||||
assert!(
|
||||
error["message"].as_str().unwrap().contains("capacity"),
|
||||
"{error}"
|
||||
);
|
||||
let mut drained = Box::pin(executor.drain());
|
||||
assert!(futures::poll!(&mut drained).is_pending());
|
||||
release.0.take().unwrap().send(()).unwrap();
|
||||
observed(drained).await.unwrap();
|
||||
// The accepted request finishes its real lookup after cancellation; the
|
||||
// normal domain error and reusable capacity survive without SQL resending.
|
||||
let after = observed(fixture.inner_router().oneshot(request()))
|
||||
.await
|
||||
.unwrap();
|
||||
response_json(after, StatusCode::NOT_FOUND).await;
|
||||
observed(executor.drain()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn assignments_page_uses_finite_db_owner_and_survives_cancel() {
|
||||
queued_caller_keeps_owner(
|
||||
"GET",
|
||||
"assignment-tracking/assignments",
|
||||
serde_json::json!({}),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn diagnostics_page_uses_finite_db_owner_and_survives_cancel() {
|
||||
queued_caller_keeps_owner(
|
||||
"GET",
|
||||
"assignment-diagnostics?assignment_id=missing",
|
||||
serde_json::json!({}),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn answer_uses_finite_host_owner_and_survives_cancel() {
|
||||
queued_caller_keeps_owner(
|
||||
"POST",
|
||||
"assignment-answer",
|
||||
serde_json::json!({
|
||||
"assignment_id":"missing", "expected_last_event_id":"missing",
|
||||
"operation_key":"answer-probe", "body":"continuing input"
|
||||
}),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -724,7 +724,7 @@ async fn catalog_case_dispatch_authenticates_and_routes_the_canonical_workspace_
|
||||
reason: ticket::TicketClosureReason::Completed,
|
||||
},
|
||||
);
|
||||
let catalog = WorkspaceServerApi::new(api.config.clone(), api.store.clone());
|
||||
let catalog = WorkspaceServerApi::new_for_test(api.config.clone(), api.store.clone());
|
||||
// Retain only this in-process fixture: do not create a real Runtime or a
|
||||
// background Orchestrator hook while testing catalog authentication/routing.
|
||||
catalog
|
||||
|
||||
@@ -529,7 +529,7 @@ async fn runtime_tag_mutations_reject_invalid_proofs_and_changed_binding_authori
|
||||
*tampered.body_mut() = Body::from(format!("{} ", input));
|
||||
runtime_tag_response(app.clone(), tampered, StatusCode::UNAUTHORIZED).await;
|
||||
}
|
||||
let store = SqliteWorkspaceStore::open(&api.config.database_path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap();
|
||||
let original = api
|
||||
.store
|
||||
.get_workspace_runtime_binding(TEST_WORKSPACE_ID, "runtime-tags")
|
||||
|
||||
@@ -47,7 +47,10 @@ pub(super) fn pre_error(error: impl Into<ApiError>) -> Failure {
|
||||
fn mutation_error(error: Error) -> Failure {
|
||||
// Storage/service failures at a mutation boundary may include a commit
|
||||
// acknowledgement failure. Never describe those as a guaranteed rollback.
|
||||
if matches!(&error, Error::Sqlite(_) | Error::Store(_)) {
|
||||
if matches!(
|
||||
&error,
|
||||
Error::Sqlite(_) | Error::Store(_) | Error::DbOperationOutcomeUnknown
|
||||
) {
|
||||
unknown()
|
||||
} else {
|
||||
pre_error(error)
|
||||
@@ -59,7 +62,7 @@ fn unknown() -> Failure {
|
||||
..fail(500, "outcome_unknown")
|
||||
}
|
||||
}
|
||||
pub(super) fn identity(
|
||||
pub(super) async fn identity(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
workspace_id: &str,
|
||||
@@ -67,7 +70,9 @@ pub(super) fn identity(
|
||||
validate_workspace_scope(api, workspace_id).map_err(pre_error)?;
|
||||
let headers =
|
||||
current_worker_contract_headers(context).map_err(|_| fail(403, "access_denied"))?;
|
||||
current_worker_identity(api, workspace_id, &headers).map_err(pre_error)
|
||||
authenticate_worker_mutation_source_owned(api, workspace_id, &headers)
|
||||
.await
|
||||
.map_err(pre_error)
|
||||
}
|
||||
async fn lock(
|
||||
api: &WorkspaceApi,
|
||||
@@ -186,9 +191,23 @@ pub(super) async fn get(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ConfigResult<Option<WorkspaceConfigAttachment>> {
|
||||
let _guard = lock(api, worker).await?;
|
||||
let grant = grant_for_worker(api, worker)?;
|
||||
Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true)))
|
||||
let guard = lock(api, worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let worker = worker.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
let grant = grant_for_worker(&api, &worker)?;
|
||||
Ok::<_, Failure>(
|
||||
active_link(&api, &worker, &grant, None)?
|
||||
.map(|link| attachment(&grant, &link, true)),
|
||||
)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
}
|
||||
pub(super) async fn attach(
|
||||
api: &WorkspaceApi,
|
||||
@@ -198,46 +217,59 @@ pub(super) async fn attach(
|
||||
if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) {
|
||||
return Err(fail(400, "invalid_alias"));
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let grant = grant_for_worker(api, worker)?;
|
||||
let access = request.access.unwrap_or(grant.access);
|
||||
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if let Some(mut link) = active_link(api, worker, &grant, None)? {
|
||||
// Same active alias/identity is idempotent; explicit write never silently downgrades.
|
||||
if request.access == Some(Access::ReadWrite)
|
||||
&& effective_access(&grant, &link) != Access::ReadWrite
|
||||
{
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly {
|
||||
link.capabilities = access_capabilities(access);
|
||||
link = api
|
||||
.store
|
||||
.attach_worker_workdir(&link)
|
||||
.map_err(mutation_error)?;
|
||||
}
|
||||
return Ok(attachment(&grant, &link, true));
|
||||
}
|
||||
let link = api
|
||||
.store
|
||||
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
|
||||
connection_id: String::new(),
|
||||
workspace_id: grant.workspace_id.clone(),
|
||||
worker: worker.clone(),
|
||||
workdir_id: grant.working_directory_id.clone(),
|
||||
alias: ALIAS.into(),
|
||||
capabilities: access_capabilities(access),
|
||||
linked_at: now_registry_timestamp(),
|
||||
unlinked_at: None,
|
||||
let guard = lock(api, worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let worker = worker.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
let grant = grant_for_worker(&api, &worker)?;
|
||||
let access = request.access.unwrap_or(grant.access);
|
||||
if access == Access::ReadWrite && grant.access != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if let Some(mut link) = active_link(&api, &worker, &grant, None)? {
|
||||
// Same active alias/identity is idempotent; explicit write never silently downgrades.
|
||||
if request.access == Some(Access::ReadWrite)
|
||||
&& effective_access(&grant, &link) != Access::ReadWrite
|
||||
{
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if access == Access::ReadOnly
|
||||
&& effective_access(&grant, &link) != Access::ReadOnly
|
||||
{
|
||||
link.capabilities = access_capabilities(access);
|
||||
link = api
|
||||
.store
|
||||
.attach_worker_workdir(&link)
|
||||
.map_err(mutation_error)?;
|
||||
}
|
||||
return Ok(attachment(&grant, &link, true));
|
||||
}
|
||||
let link = api
|
||||
.store
|
||||
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
|
||||
connection_id: String::new(),
|
||||
workspace_id: grant.workspace_id.clone(),
|
||||
worker: worker.clone(),
|
||||
workdir_id: grant.working_directory_id.clone(),
|
||||
alias: ALIAS.into(),
|
||||
capabilities: access_capabilities(access),
|
||||
linked_at: now_registry_timestamp(),
|
||||
unlinked_at: None,
|
||||
})
|
||||
.map_err(mutation_error)?;
|
||||
// No Runtime filesystem/session synchronization for a logical config target.
|
||||
api.worker_projection
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(attachment(&grant, &link, false))
|
||||
})())
|
||||
})
|
||||
.map_err(mutation_error)?;
|
||||
// No Runtime filesystem/session synchronization for a logical config target.
|
||||
api.worker_projection
|
||||
.refresh(worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(attachment(&grant, &link, false))
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
fn resolve(
|
||||
api: &WorkspaceApi,
|
||||
@@ -313,9 +345,22 @@ pub(super) async fn observe(
|
||||
for path in &request.paths {
|
||||
path_valid(path, true)?;
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, _guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -427,9 +472,22 @@ pub(super) async fn read(
|
||||
request: WorkspaceConfigReadRequest,
|
||||
) -> ConfigResult<WorkspaceConfigReadResponse> {
|
||||
path_valid(&request.path, false)?;
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, _guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -482,12 +540,25 @@ pub(super) async fn commit(
|
||||
server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?,
|
||||
}
|
||||
}
|
||||
let _guard = lock(api, worker).await?;
|
||||
let (grant, link) = resolve(api, worker, &request.connection_id)?;
|
||||
if effective_access(&grant, &link) != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
let state = tree(api)?;
|
||||
let guard = lock(api, worker).await?;
|
||||
let resolution_api = api.clone();
|
||||
let resolution_worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let (grant, link, state, guard) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (grant, link) = resolve(&resolution_api, &resolution_worker, &connection_id)?;
|
||||
if effective_access(&grant, &link) != Access::ReadWrite {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
let state = tree(&resolution_api)?;
|
||||
Ok::<_, Failure>((grant, link, state, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
let schema_fingerprint = api
|
||||
.config_schema_registry
|
||||
.compose()
|
||||
@@ -501,17 +572,19 @@ pub(super) async fn commit(
|
||||
// Evaluation/semantic validation are pure preparation. Bound this stage
|
||||
// without ever dispatching persistence in a timed-out background task.
|
||||
let preparation_api = api.clone();
|
||||
let preparation = tokio::task::spawn_blocking(move || {
|
||||
prepare_workspace_config_tree(
|
||||
let executor = api.store.db_executor();
|
||||
let preparation = executor.run(move || {
|
||||
// Timeout/cancellation releases exclusion only after actual preparation ends.
|
||||
let candidate = prepare_workspace_config_tree(
|
||||
&preparation_api,
|
||||
&preparation_api.config.workspace_id,
|
||||
&canonical,
|
||||
)
|
||||
)?;
|
||||
Ok::<_, Error>((candidate, guard))
|
||||
});
|
||||
let candidate = tokio::time::timeout(DEADLINE, preparation)
|
||||
let (candidate, guard) = tokio::time::timeout(DEADLINE, preparation)
|
||||
.await
|
||||
.map_err(|_| fail(503, "evaluation_timeout"))?
|
||||
.map_err(|_| fail(500, "evaluation_unavailable"))?
|
||||
.map_err(pre_error)?;
|
||||
if candidate.base_digest != state.snapshot.digest
|
||||
|| candidate.base_toolchain_fingerprint != state.contract.fingerprint
|
||||
@@ -520,15 +593,78 @@ pub(super) async fn commit(
|
||||
{
|
||||
return Err(fail(409, "stale_validator"));
|
||||
}
|
||||
// Recheck durable authority before the first auxiliary/canonical effects.
|
||||
let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?;
|
||||
if current_grant != grant || current_link.capabilities != link.capabilities {
|
||||
return Err(fail(409, "authority_changed"));
|
||||
let persistence_api = api.clone();
|
||||
let worker = worker.clone();
|
||||
let connection_id = request.connection_id.clone();
|
||||
let expected_grant = grant.clone();
|
||||
let expected_link = link.clone();
|
||||
let (state, publication) = api
|
||||
.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let (current_grant, current_link) =
|
||||
resolve(&persistence_api, &worker, &connection_id)?;
|
||||
if current_grant != expected_grant
|
||||
|| current_link.capabilities != expected_link.capabilities
|
||||
{
|
||||
return Err(fail(409, "authority_changed"));
|
||||
}
|
||||
// Preserve the shared helper's ordering and atomic CAS, but never
|
||||
// run its remote Runtime publication on the sole DB worker.
|
||||
project_repository_access_candidate(
|
||||
&*persistence_api.store,
|
||||
&persistence_api.repository_secrets,
|
||||
&persistence_api.config.workspace_id,
|
||||
&candidate,
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
let state = persistence_api
|
||||
.config_store
|
||||
.commit_evaluated_workspace_config(
|
||||
&persistence_api.config.workspace_id,
|
||||
&candidate,
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
let publication = persistence_api
|
||||
.prompt_projection_cache
|
||||
.resolve(&persistence_api.config.workspace_id, &state)
|
||||
.ok()
|
||||
.and_then(|projection| {
|
||||
// Dispatch from the actual persistence owner even if its
|
||||
// waiter was cancelled. Runtime I/O owns exclusion, not
|
||||
// a DB worker or an open SQL transaction.
|
||||
let (complete, publication) = tokio::sync::oneshot::channel();
|
||||
let runtime = persistence_api.runtime.clone();
|
||||
let admission = persistence_api.host_executors.host().and_then(|executor| {
|
||||
executor.dispatch(move || {
|
||||
let _guard = guard;
|
||||
let _diagnostics = runtime
|
||||
.observe_workspace_prompt_projection((*projection).clone());
|
||||
let _ = complete.send(());
|
||||
})
|
||||
});
|
||||
match admission {
|
||||
Ok(()) => Some(publication),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
workspace_id = %persistence_api.config.workspace_id,
|
||||
%error,
|
||||
"Workspace config committed but Runtime publication was not admitted"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok((state, publication))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)??;
|
||||
if let Some(publication) = publication {
|
||||
// Publication is best effort and cannot turn an acknowledged CAS into an error.
|
||||
let _ = publication.await;
|
||||
}
|
||||
// From this boundary repository credential projection may have side effects.
|
||||
// Never falsely claim rollback or automatically replay a failed/uncertain save.
|
||||
let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(WorkspaceConfigCommitResponse {
|
||||
validator: validator(&grant, &link, &state, &schema_fingerprint, ""),
|
||||
digest: state.snapshot.digest,
|
||||
@@ -544,37 +680,61 @@ pub(super) async fn create_grant(
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id);
|
||||
let _guard = lock(api, &worker).await?;
|
||||
let summary = api
|
||||
.runtime
|
||||
.worker(&worker)
|
||||
.map_err(|_| fail(404, "worker_not_found"))?;
|
||||
let guard = lock(api, &worker).await?;
|
||||
let runtime = api.runtime.clone();
|
||||
let lookup_worker = worker.clone();
|
||||
let (summary, guard) = api
|
||||
.host_executors
|
||||
.host()
|
||||
.map_err(pre_error)?
|
||||
.run(move || {
|
||||
Ok::<_, Error>((|| {
|
||||
let summary = runtime
|
||||
.worker(&lookup_worker)
|
||||
.map_err(|_| fail(404, "worker_not_found"))?;
|
||||
Ok::<_, Failure>((summary, guard))
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)??;
|
||||
if summary.workspace.workspace_id.as_deref() != Some(workspace_id) {
|
||||
return Err(fail(403, "access_denied"));
|
||||
}
|
||||
if let Some(grant) = api
|
||||
.store
|
||||
.current_workspace_config_grant(workspace_id, &worker)
|
||||
.map_err(pre_error)?
|
||||
{
|
||||
if grant.access != request.access {
|
||||
return Err(fail(409, "grant_exists_revoke_first"));
|
||||
}
|
||||
return Ok(grant);
|
||||
}
|
||||
let grant = WorkspaceConfigGrantResponse {
|
||||
grant_id: Uuid::now_v7().to_string(),
|
||||
workspace_id: workspace_id.into(),
|
||||
runtime_id: worker.runtime_id,
|
||||
worker_id: worker.worker_id,
|
||||
working_directory_id: Uuid::now_v7().to_string(),
|
||||
access: request.access,
|
||||
revoked: false,
|
||||
};
|
||||
api.store
|
||||
.create_workspace_config_grant(&grant, &actor.account_id)
|
||||
.map_err(mutation_error)?;
|
||||
Ok(grant)
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let account_id = actor.account_id.clone();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
if let Some(grant) = api
|
||||
.store
|
||||
.current_workspace_config_grant(&workspace_id, &worker)
|
||||
.map_err(pre_error)?
|
||||
{
|
||||
if grant.access != request.access {
|
||||
return Err(fail(409, "grant_exists_revoke_first"));
|
||||
}
|
||||
return Ok(grant);
|
||||
}
|
||||
let grant = WorkspaceConfigGrantResponse {
|
||||
grant_id: Uuid::now_v7().to_string(),
|
||||
workspace_id,
|
||||
runtime_id: worker.runtime_id,
|
||||
worker_id: worker.worker_id,
|
||||
working_directory_id: Uuid::now_v7().to_string(),
|
||||
access: request.access,
|
||||
revoked: false,
|
||||
};
|
||||
api.store
|
||||
.create_workspace_config_grant(&grant, &account_id)
|
||||
.map_err(mutation_error)?;
|
||||
Ok(grant)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
pub(super) async fn revoke_grant(
|
||||
api: &WorkspaceApi,
|
||||
@@ -585,39 +745,60 @@ pub(super) async fn revoke_grant(
|
||||
require_workspace_owner(api, workspace_id, actor, "Workspace config grants")
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
let lookup_api = api.clone();
|
||||
let lookup_workspace_id = workspace_id.to_owned();
|
||||
let lookup_grant_id = grant_id.to_owned();
|
||||
let mut grant = api
|
||||
.store
|
||||
.get_workspace_config_grant(workspace_id, grant_id)
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
lookup_api
|
||||
.store
|
||||
.get_workspace_config_grant(&lookup_workspace_id, &lookup_grant_id)
|
||||
})
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
.ok_or_else(|| fail(404, "not_found"))?;
|
||||
let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id);
|
||||
let _guard = lock(api, &worker).await?;
|
||||
api.store
|
||||
.revoke_workspace_config_grant(workspace_id, grant_id)
|
||||
.map_err(mutation_error)?;
|
||||
grant.revoked = true;
|
||||
// Revocation is durable before unlink; a failed cleanup cannot restore access.
|
||||
if let Some(link) = api
|
||||
.store
|
||||
.list_workdir_worker_links(workspace_id, &grant.working_directory_id)
|
||||
.map_err(|_| unknown())?
|
||||
.into_iter()
|
||||
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
|
||||
{
|
||||
api.store
|
||||
.detach_worker_workdir_connection(
|
||||
workspace_id,
|
||||
&worker,
|
||||
&link.alias,
|
||||
&link.connection_id,
|
||||
&now_registry_timestamp(),
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
}
|
||||
api.worker_projection
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(grant)
|
||||
let guard = lock(api, &worker).await?;
|
||||
let executor = api.store.db_executor();
|
||||
let api = api.clone();
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let grant_id = grant_id.to_owned();
|
||||
executor
|
||||
.run(move || {
|
||||
let _guard = guard;
|
||||
Ok::<_, Error>((|| {
|
||||
api.store
|
||||
.revoke_workspace_config_grant(&workspace_id, &grant_id)
|
||||
.map_err(mutation_error)?;
|
||||
grant.revoked = true;
|
||||
// Revocation is durable before unlink; a failed cleanup cannot restore access.
|
||||
if let Some(link) = api
|
||||
.store
|
||||
.list_workdir_worker_links(&workspace_id, &grant.working_directory_id)
|
||||
.map_err(|_| unknown())?
|
||||
.into_iter()
|
||||
.find(|link| link.unlinked_at.is_none() && link.worker == worker)
|
||||
{
|
||||
api.store
|
||||
.detach_worker_workdir_connection(
|
||||
&workspace_id,
|
||||
&worker,
|
||||
&link.alias,
|
||||
&link.connection_id,
|
||||
&now_registry_timestamp(),
|
||||
)
|
||||
.map_err(|_| unknown())?;
|
||||
}
|
||||
api.worker_projection
|
||||
.refresh(&worker)
|
||||
.map_err(|_| unknown())?;
|
||||
Ok(grant)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
.map_err(mutation_error)?
|
||||
}
|
||||
|
||||
pub(super) fn summary(
|
||||
@@ -655,7 +836,11 @@ pub(super) fn summary(
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn workspace_config_mutation_storage_failures_are_conservatively_unknown() {
|
||||
fn workspace_config_mutation_storage_or_executor_failures_are_conservatively_unknown() {
|
||||
assert_eq!(
|
||||
mutation_error(Error::DbOperationOutcomeUnknown).classification,
|
||||
Classification::Unknown
|
||||
);
|
||||
assert_eq!(
|
||||
mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification,
|
||||
Classification::Unknown
|
||||
|
||||
@@ -318,7 +318,7 @@ mod workspace_config_integration {
|
||||
};
|
||||
for scoped in [false, true] {
|
||||
let app = if scoped {
|
||||
workspace_server_router(WorkspaceServerApi::new(
|
||||
workspace_server_router(WorkspaceServerApi::new_for_test(
|
||||
api.config.clone(),
|
||||
api.store.clone(),
|
||||
))
|
||||
@@ -803,9 +803,9 @@ mod workspace_config_integration {
|
||||
.supports(workdir::WorkdirSessionCapability::Command)
|
||||
);
|
||||
let lock = current_worker_session_lock(api, &fixture.worker);
|
||||
let _guard = lock.lock().await;
|
||||
let _guard = Arc::new(lock.lock_owned().await);
|
||||
assert!(
|
||||
open_current_worker_workdir_session_locked(api, &fixture.worker, &link)
|
||||
open_current_worker_workdir_session_locked(api, &fixture.worker, &link, &_guard)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
@@ -1385,7 +1385,7 @@ mod workspace_config_integration {
|
||||
let api = &fixture.api;
|
||||
let granted = grant(api, &fixture.worker, Access::ReadWrite).await;
|
||||
let attached = attach(api, &fixture.worker).await;
|
||||
let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap();
|
||||
let restarted = SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap();
|
||||
assert_eq!(
|
||||
restarted
|
||||
.get_workspace_config_grant(TEST_WORKSPACE_ID, &granted.grant_id)
|
||||
|
||||
@@ -800,7 +800,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
||||
let base = format!("http://{addr}");
|
||||
let mut config = test_server_config(root.path());
|
||||
config.backend_base_url = Some("server-test".into());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
let captured = Arc::new(Mutex::new(None));
|
||||
let abort = Arc::new(Mutex::new(None));
|
||||
let factory = ConfigCaptureFactory {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -263,7 +263,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64
|
||||
let tags_before = retained.as_ref().map(|_| classification_rows(&conn));
|
||||
let auth_before = fixture_auth_rows(&conn);
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?,93);
|
||||
assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11);
|
||||
@@ -490,7 +490,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history
|
||||
let metadata_before = stored_fixture_rows(&conn, "typed_ticket_case_metadata");
|
||||
let auth_before = fixture_auth_rows(&conn);
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open(&path).unwrap();
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?,93);
|
||||
|
||||
@@ -750,7 +750,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn retry_keeps_resolved_config_and_credential_candidates_after_fallback_moves() {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
futures::executor::block_on(store.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace".to_string(),
|
||||
owner_account_id: "owner-account".to_string(),
|
||||
|
||||
@@ -925,7 +925,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn seeded_store() -> (SqliteWorkspaceStore, WorkdirRegistryRecord) {
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-a".to_string(),
|
||||
|
||||
@@ -5,7 +5,7 @@ use chrono::Utc;
|
||||
use protocol::subscription::{
|
||||
EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot,
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
use tokio::sync::{Semaphore, broadcast};
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use crate::runtime_subscription::{
|
||||
@@ -59,7 +59,14 @@ pub struct WorkerProjectionService {
|
||||
events: broadcast::Sender<WorkerProjectionEvent>,
|
||||
ordering: Mutex<()>,
|
||||
attach_gate: tokio::sync::Mutex<()>,
|
||||
tasks: Mutex<HashMap<String, JoinHandle<()>>>,
|
||||
tasks: Mutex<HashMap<String, RuntimeProjectionLifetime>>,
|
||||
}
|
||||
|
||||
struct RuntimeProjectionLifetime {
|
||||
consumer: Option<JoinHandle<()>>,
|
||||
// The permit belongs to the actual DB closure, not its cancellable async waiter.
|
||||
// Keep this owner in the map even if attachment initialization is cancelled.
|
||||
operations: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl WorkerProjectionService {
|
||||
@@ -75,17 +82,89 @@ impl WorkerProjectionService {
|
||||
})
|
||||
}
|
||||
|
||||
/// Stops consumers but does not claim that accepted database work has finished.
|
||||
/// Async shutdown callers must use `shutdown_async` to drain accepted operations.
|
||||
pub fn shutdown(&self) {
|
||||
for (_, task) in self
|
||||
for lifetime in self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.drain()
|
||||
.values()
|
||||
{
|
||||
task.abort();
|
||||
if let Some(task) = &lifetime.consumer {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn shutdown_async(&self) {
|
||||
let _attach = self.attach_gate.lock().await;
|
||||
self.shutdown();
|
||||
let runtime_ids = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for runtime_id in runtime_ids {
|
||||
self.drain_runtime(&runtime_id).await;
|
||||
}
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.clear();
|
||||
}
|
||||
|
||||
async fn drain_runtime(&self, runtime_id: &str) {
|
||||
let previous = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.get_mut(runtime_id)
|
||||
.map(|lifetime| (lifetime.consumer.take(), lifetime.operations.clone()));
|
||||
if let Some((consumer, operations)) = previous {
|
||||
if let Some(consumer) = consumer {
|
||||
consumer.abort();
|
||||
let _ = consumer.await;
|
||||
}
|
||||
// Aborting the waiter cannot abort a queued/running synchronous job.
|
||||
// Cancellation here leaves the gate in the map for the next drain.
|
||||
let _drained = operations
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("operation gate closed");
|
||||
}
|
||||
}
|
||||
|
||||
// Caller holds attach_gate. Install ownership before any initialization await.
|
||||
async fn replace_runtime(&self, runtime_id: &str) -> Arc<Semaphore> {
|
||||
self.drain_runtime(runtime_id).await;
|
||||
let operations = Arc::new(Semaphore::new(1));
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.insert(
|
||||
runtime_id.to_string(),
|
||||
RuntimeProjectionLifetime {
|
||||
consumer: None,
|
||||
operations: operations.clone(),
|
||||
},
|
||||
);
|
||||
operations
|
||||
}
|
||||
|
||||
pub async fn subscribe_ordered_async(
|
||||
self: &Arc<Self>,
|
||||
limit: usize,
|
||||
) -> crate::Result<WorkerProjectionSubscription> {
|
||||
let service = self.clone();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.subscribe_ordered(limit))
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn subscribe_ordered(&self, limit: usize) -> crate::Result<WorkerProjectionSubscription> {
|
||||
let _ordering = self
|
||||
.ordering
|
||||
@@ -125,19 +204,10 @@ impl WorkerProjectionService {
|
||||
broker: &RuntimeSubscriptionBroker,
|
||||
runtime_id: &str,
|
||||
) -> Result<(), RuntimeSubscriptionBrokerError> {
|
||||
// Serialize replacement and wait for the old consumer to stop before the new
|
||||
// lifetime can apply its snapshot. Late notifications can therefore never
|
||||
// cross a Runtime ownership boundary.
|
||||
// Serialize replacement, stop admission by aborting the consumer, and then
|
||||
// drain ownership held by the actual SQL jobs before the new snapshot.
|
||||
let _attach = self.attach_gate.lock().await;
|
||||
let previous = self
|
||||
.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.remove(runtime_id);
|
||||
if let Some(previous) = previous {
|
||||
previous.abort();
|
||||
let _ = previous.await;
|
||||
}
|
||||
let operations = self.replace_runtime(runtime_id).await;
|
||||
|
||||
let mut subscription =
|
||||
broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?;
|
||||
@@ -148,12 +218,25 @@ impl WorkerProjectionService {
|
||||
break;
|
||||
};
|
||||
let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. });
|
||||
if let Err(error) = self.apply_broker_event(runtime_id, event) {
|
||||
if let Err(error) = self
|
||||
.apply_broker_event_owned_async(runtime_id, event, operations.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
runtime_id,
|
||||
error = %error,
|
||||
"failed to initialize durable Worker projection; restarting from a fresh snapshot"
|
||||
);
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbCapacityExceeded
|
||||
) {
|
||||
// Do not automatically replay a possibly committed mutation or
|
||||
// spin on executor overload; attachment must be retried explicitly.
|
||||
return Err(RuntimeSubscriptionBrokerError::Closed);
|
||||
}
|
||||
subscription.restart()?;
|
||||
continue;
|
||||
}
|
||||
@@ -169,12 +252,27 @@ impl WorkerProjectionService {
|
||||
let Some(service) = service.upgrade() else {
|
||||
return;
|
||||
};
|
||||
if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) {
|
||||
if let Err(error) = service
|
||||
.apply_broker_event_owned_async(
|
||||
task_runtime_id.as_str(),
|
||||
event,
|
||||
operations.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
runtime_id = %task_runtime_id,
|
||||
error = %error,
|
||||
"failed to update durable Worker projection; restarting from a fresh snapshot"
|
||||
);
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbCapacityExceeded
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if subscription.restart().is_err() {
|
||||
return;
|
||||
}
|
||||
@@ -184,10 +282,61 @@ impl WorkerProjectionService {
|
||||
self.tasks
|
||||
.lock()
|
||||
.expect("Worker projection task lock poisoned")
|
||||
.insert(runtime_id, task);
|
||||
.get_mut(&runtime_id)
|
||||
.expect("Runtime lifetime installed before initialization")
|
||||
.consumer = Some(task);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn seed_observation_async(
|
||||
self: &Arc<Self>,
|
||||
runtime_id: &str,
|
||||
worker: &protocol::subscription::SubscriptionWorker,
|
||||
observed_at: &str,
|
||||
) -> crate::Result<()> {
|
||||
let service = self.clone();
|
||||
let runtime_id = runtime_id.to_string();
|
||||
let worker = worker.clone();
|
||||
let observed_at = observed_at.to_string();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.seed_observation(&runtime_id, &worker, &observed_at))
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn refresh_async(self: &Arc<Self>, worker: &RuntimeWorkerRef) -> crate::Result<()> {
|
||||
let service = self.clone();
|
||||
let worker = worker.clone();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || service.refresh(&worker))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn apply_broker_event_owned_async(
|
||||
self: &Arc<Self>,
|
||||
runtime_id: &str,
|
||||
event: BrokerSubscriptionEvent,
|
||||
operations: Arc<Semaphore>,
|
||||
) -> crate::Result<()> {
|
||||
let operation = operations
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("operation gate closed");
|
||||
let service = self.clone();
|
||||
let runtime_id = runtime_id.to_string();
|
||||
self.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
// The permit must outlive SQL commit AND ordered publication, even
|
||||
// when the executor result receiver has been dropped by cancellation.
|
||||
let _operation = operation;
|
||||
service.apply_broker_event(&runtime_id, event)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn seed_observation(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -313,7 +462,11 @@ mod tests {
|
||||
tempfile::TempDir,
|
||||
) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap());
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open_for_test(temp.path().join("workspace.db"))
|
||||
.unwrap()
|
||||
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".to_string(),
|
||||
@@ -355,6 +508,266 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn observation(
|
||||
state: protocol::subscription::SubscriptionWorkerState,
|
||||
) -> protocol::subscription::SubscriptionWorker {
|
||||
protocol::subscription::SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a").unwrap(),
|
||||
runtime_id: Some("runtime-a".to_string()),
|
||||
resource_key: None,
|
||||
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
|
||||
worker_state: None,
|
||||
state,
|
||||
has_running_internal_workers: false,
|
||||
workspace_id: Some(WORKSPACE_ID.to_string()),
|
||||
display_name: None,
|
||||
profile: None,
|
||||
job: None,
|
||||
workdir_attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// Real second-connection contention; the controller releases automatically on
|
||||
// failure, so a regression that blocks the Tokio thread cannot hang the suite.
|
||||
struct WriteLock {
|
||||
release: Option<std::sync::mpsc::Sender<()>>,
|
||||
controller: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl WriteLock {
|
||||
fn acquire(path: std::path::PathBuf) -> Self {
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
let (locked, ready) = std::sync::mpsc::channel();
|
||||
let controller = std::thread::spawn(move || {
|
||||
let connection = rusqlite::Connection::open(path).unwrap();
|
||||
connection.execute_batch("BEGIN IMMEDIATE").unwrap();
|
||||
locked.send(()).unwrap();
|
||||
let _ = released.recv_timeout(std::time::Duration::from_secs(3));
|
||||
connection.execute_batch("ROLLBACK").unwrap();
|
||||
});
|
||||
ready.recv().unwrap();
|
||||
Self {
|
||||
release: Some(release),
|
||||
controller: Some(controller),
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&mut self) {
|
||||
if let Some(release) = self.release.take() {
|
||||
let _ = release.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WriteLock {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
if let Some(controller) = self.controller.take() {
|
||||
controller.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_sql(service: &WorkerProjectionService) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
// With the second connection holding the write lock, a held ordering
|
||||
// gate identifies the accepted synchronous mutation, not a fake future.
|
||||
while service.ordering.try_lock().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("projection job did not enter synchronous SQL");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_write_contention_keeps_projection_timer_and_control_live_and_publishes_committed_payload()
|
||||
{
|
||||
use protocol::subscription::SubscriptionWorkerState;
|
||||
let (store, service, temp) = fixture().await;
|
||||
store
|
||||
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
|
||||
.unwrap();
|
||||
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
|
||||
assert_eq!(subscription.take_snapshot().len(), 1);
|
||||
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
|
||||
let publishing_service = service.clone();
|
||||
let publication = tokio::spawn(async move {
|
||||
publishing_service
|
||||
.seed_observation_async(
|
||||
"runtime-a",
|
||||
&observation(SubscriptionWorkerState::Idle),
|
||||
"2",
|
||||
)
|
||||
.await
|
||||
});
|
||||
wait_for_sql(&service).await;
|
||||
let started = std::time::Instant::now();
|
||||
let (control, controlled) = tokio::sync::oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
control.send("control progressed").unwrap();
|
||||
});
|
||||
let (_, control_result) = tokio::join!(
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)),
|
||||
controlled,
|
||||
);
|
||||
assert_eq!(control_result.unwrap(), "control progressed");
|
||||
assert!(
|
||||
started.elapsed() < std::time::Duration::from_secs(1),
|
||||
"SQLite blocked the current-thread executor"
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
lock.release();
|
||||
publication.await.unwrap().unwrap();
|
||||
let committed = subscription.recv().await.unwrap();
|
||||
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
|
||||
let snapshot = snapshot.take_snapshot();
|
||||
assert_eq!(
|
||||
committed.changes,
|
||||
vec![WorkerCatalogChange::Upsert(snapshot[0].clone())]
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot[0].observation.as_ref().unwrap().worker.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
service
|
||||
.seed_observation_async(
|
||||
"runtime-a",
|
||||
&observation(SubscriptionWorkerState::Running),
|
||||
"3",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Delayed readers receive the first transaction's payload, not a re-query
|
||||
// of mutable authority after a later commit.
|
||||
let WorkerCatalogChange::Upsert(first) = &committed.changes[0] else {
|
||||
panic!("expected upsert");
|
||||
};
|
||||
assert_eq!(
|
||||
first.observation.as_ref().unwrap().worker.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
service
|
||||
.refresh_async(&RuntimeWorkerRef::new("runtime-a", "worker-a"))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn cancelled_runtime_consumer_and_replacement_drain_real_sql_before_new_snapshot() {
|
||||
use crate::runtime_subscription::BrokerSubscriptionEvent;
|
||||
use protocol::subscription::SubscriptionWorkerState;
|
||||
let (store, service, temp) = fixture().await;
|
||||
store
|
||||
.upsert_worker_registry(&worker("runtime-a", "worker-a"))
|
||||
.unwrap();
|
||||
let mut subscription = service.subscribe_ordered_async(10).await.unwrap();
|
||||
subscription.take_snapshot();
|
||||
let operations = service.replace_runtime("runtime-a").await;
|
||||
let old_operations = operations.clone();
|
||||
let mut lock = WriteLock::acquire(temp.path().join("workspace.db"));
|
||||
let old_service = service.clone();
|
||||
let consumer = tokio::spawn(async move {
|
||||
old_service
|
||||
.apply_broker_event_owned_async(
|
||||
"runtime-a",
|
||||
BrokerSubscriptionEvent::Event {
|
||||
payload: protocol::subscription::SubscriptionEventPayload::WorkerUpserted {
|
||||
worker: observation(SubscriptionWorkerState::Running),
|
||||
},
|
||||
},
|
||||
operations,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
service
|
||||
.tasks
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("runtime-a")
|
||||
.unwrap()
|
||||
.consumer = Some(consumer);
|
||||
wait_for_sql(&service).await;
|
||||
let replacing_service = service.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
let _attach = replacing_service.attach_gate.lock().await;
|
||||
replacing_service.replace_runtime("runtime-a").await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
replacement.abort();
|
||||
assert!(replacement.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(
|
||||
old_operations.available_permits(),
|
||||
0,
|
||||
"cancelled drain lost accepted SQL ownership"
|
||||
);
|
||||
// A second replacement must still observe the old gate after cancellation.
|
||||
let replacing_service = service.clone();
|
||||
let mut replacement = tokio::spawn(async move {
|
||||
let _attach = replacing_service.attach_gate.lock().await;
|
||||
let operations = replacing_service.replace_runtime("runtime-a").await;
|
||||
replacing_service
|
||||
.apply_broker_event_owned_async(
|
||||
"runtime-a",
|
||||
BrokerSubscriptionEvent::Snapshot {
|
||||
snapshot: protocol::subscription::SubscriptionSnapshot::Workers {
|
||||
workers: vec![observation(SubscriptionWorkerState::Idle)],
|
||||
},
|
||||
},
|
||||
operations,
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(20), &mut replacement)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
lock.release();
|
||||
replacement.await.unwrap().unwrap();
|
||||
assert_eq!(old_operations.available_permits(), 1);
|
||||
let old_commit = subscription.recv().await.unwrap();
|
||||
let new_commit = subscription.recv().await.unwrap();
|
||||
for (commit, expected) in [
|
||||
(old_commit, SubscriptionWorkerState::Running),
|
||||
(new_commit, SubscriptionWorkerState::Idle),
|
||||
] {
|
||||
let [WorkerCatalogChange::Upsert(record)] = commit.changes.as_slice() else {
|
||||
panic!("expected one committed upsert");
|
||||
};
|
||||
assert_eq!(record.observation.as_ref().unwrap().worker.state, expected);
|
||||
}
|
||||
let mut snapshot = service.subscribe_ordered_async(10).await.unwrap();
|
||||
assert_eq!(
|
||||
snapshot.take_snapshot()[0]
|
||||
.observation
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.worker
|
||||
.state,
|
||||
SubscriptionWorkerState::Idle
|
||||
);
|
||||
assert!(matches!(
|
||||
subscription.try_recv(),
|
||||
Err(tokio::sync::broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
service.shutdown_async().await;
|
||||
assert!(service.tasks.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn snapshot_registration_is_atomic_with_concurrent_publication() {
|
||||
let (store, service, _temp) = fixture().await;
|
||||
|
||||
@@ -70,10 +70,24 @@ pub enum WorkerMutationSourceProofError {
|
||||
Replay,
|
||||
#[error("source Worker is not a current member of this Workspace Runtime catalog")]
|
||||
WorkerCatalogMembership,
|
||||
#[error("source proof authority is unavailable")]
|
||||
Unavailable,
|
||||
#[error("source proof authority failed: {0}")]
|
||||
Authority(String),
|
||||
}
|
||||
|
||||
impl From<crate::Error> for WorkerMutationSourceProofError {
|
||||
fn from(error: crate::Error) -> Self {
|
||||
match error {
|
||||
crate::Error::DbCapacityExceeded
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::Sqlite(_) => Self::Unavailable,
|
||||
other => Self::Authority(other.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn verify_worker_remove_source(
|
||||
api: &WorkspaceApi,
|
||||
proof: PresentedWorkerMutationSourceProof<'_>,
|
||||
@@ -107,7 +121,7 @@ async fn verify_worker_remove_source_with(
|
||||
let trusted = store
|
||||
.get_workspace_runtime_binding(&config.workspace_id, &unverified.iss)
|
||||
.await
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?
|
||||
.map_err(WorkerMutationSourceProofError::from)?
|
||||
.filter(|record| record.revoked_at.is_none())
|
||||
.ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?;
|
||||
let public_key = trusted.public_key.as_str();
|
||||
@@ -151,15 +165,23 @@ async fn verify_worker_remove_source_with(
|
||||
runtime_id: claims.iss.clone(),
|
||||
worker_id: claims.worker_id.clone(),
|
||||
};
|
||||
let member = store
|
||||
.get_worker_registry(&config.workspace_id, &worker)
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
if member.is_none() {
|
||||
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
|
||||
}
|
||||
let workspace_id = config.workspace_id.clone();
|
||||
let catalog_store = Arc::clone(store);
|
||||
store
|
||||
.require_current_worker_singleton_owner(&config.workspace_id, &worker)
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
let member = catalog_store
|
||||
.get_worker_registry(&workspace_id, &worker)
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
if member.is_none() {
|
||||
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
|
||||
}
|
||||
catalog_store
|
||||
.require_current_worker_singleton_owner(&workspace_id, &worker)
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
|
||||
let consumed_at = chrono::Utc::now().to_rfc3339();
|
||||
let consumed = store
|
||||
@@ -172,7 +194,7 @@ async fn verify_worker_remove_source_with(
|
||||
&consumed_at,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
|
||||
.map_err(WorkerMutationSourceProofError::from)?;
|
||||
if !consumed {
|
||||
return Err(WorkerMutationSourceProofError::Replay);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use worker_runtime::auth::{
|
||||
RuntimeAuthError, RuntimeRequestClaim, RuntimeRequestSourceClaims,
|
||||
RuntimeRequestSourceExpectation, decode_runtime_request_source_claims,
|
||||
@@ -33,9 +35,21 @@ pub struct RuntimeRequestProofError {
|
||||
pub(crate) exp: Option<i64>,
|
||||
pub(crate) now: Option<i64>,
|
||||
pub(crate) time_delta_seconds: Option<u64>,
|
||||
/// Operational unavailability is not a proof or principal rejection.
|
||||
pub(crate) availability: bool,
|
||||
}
|
||||
|
||||
impl RuntimeRequestProofError {
|
||||
pub fn is_availability_error(&self) -> bool {
|
||||
self.availability
|
||||
}
|
||||
|
||||
fn unavailable(mut self, reason: &'static str) -> Self {
|
||||
self.reason = reason;
|
||||
self.availability = true;
|
||||
self
|
||||
}
|
||||
|
||||
fn undecoded() -> Self {
|
||||
Self {
|
||||
reason: "invalid",
|
||||
@@ -47,6 +61,7 @@ impl RuntimeRequestProofError {
|
||||
exp: None,
|
||||
now: None,
|
||||
time_delta_seconds: None,
|
||||
availability: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +121,18 @@ impl RuntimeRequestProofError {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::Error> for RuntimeRequestProofError {
|
||||
fn from(error: crate::Error) -> Self {
|
||||
let reason = match error {
|
||||
crate::Error::DbCapacityExceeded => "db_capacity_exceeded",
|
||||
crate::Error::DbExecutorUnavailable => "db_executor_unavailable",
|
||||
crate::Error::DbOperationOutcomeUnknown => "db_operation_outcome_unknown",
|
||||
_ => "database_authority_unavailable",
|
||||
};
|
||||
Self::undecoded().unavailable(reason)
|
||||
}
|
||||
}
|
||||
|
||||
/// Domain-separated fixed-length fingerprints, not raw request-derived IDs.
|
||||
/// Operators can correlate a known ID by applying this same function.
|
||||
pub fn diagnostic_id_hash(id: &str) -> String {
|
||||
@@ -130,7 +157,7 @@ pub async fn verify_runtime_request_source_proof(
|
||||
body_digest: &str,
|
||||
) -> Result<VerifiedRuntimeRequestSource, RuntimeRequestProofError> {
|
||||
verify_runtime_request_source_proof_with_store(
|
||||
api.store.as_ref(),
|
||||
Arc::clone(&api.store),
|
||||
&api.config,
|
||||
proof,
|
||||
workspace_id,
|
||||
@@ -144,7 +171,7 @@ pub async fn verify_runtime_request_source_proof(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn verify_runtime_request_source_proof_with_store(
|
||||
store: &dyn ControlPlaneStore,
|
||||
store: Arc<dyn ControlPlaneStore>,
|
||||
config: &ServerConfig,
|
||||
proof: &str,
|
||||
workspace_id: &str,
|
||||
@@ -169,7 +196,7 @@ pub async fn verify_runtime_request_source_proof_with_store(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
store: &dyn ControlPlaneStore,
|
||||
store: Arc<dyn ControlPlaneStore>,
|
||||
config: &ServerConfig,
|
||||
proof: &str,
|
||||
workspace_id: &str,
|
||||
@@ -182,15 +209,18 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
let unverified = decode_runtime_request_source_claims(proof)
|
||||
.map_err(|error| RuntimeRequestProofError::undecoded().auth_error(error))?;
|
||||
let diagnostic = RuntimeRequestProofError::decoded(&unverified);
|
||||
let audience = remote_audience(config, workspace_id)
|
||||
.map_err(|_| diagnostic.clone().reason("backend_audience_unavailable"))?;
|
||||
let audience = remote_audience(config, workspace_id).map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.unavailable("backend_audience_unavailable")
|
||||
})?;
|
||||
let trusted = store
|
||||
.get_workspace_runtime_binding(workspace_id, &unverified.iss)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("runtime_trust_authority_unavailable")
|
||||
.unavailable("runtime_trust_authority_unavailable")
|
||||
})?
|
||||
.filter(|record| record.revoked_at.is_none())
|
||||
.ok_or_else(|| {
|
||||
@@ -238,7 +268,11 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
&consumed_at,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| diagnostic.clone().reason("replay_authority_unavailable"))?
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.unavailable("replay_authority_unavailable")
|
||||
})?
|
||||
{
|
||||
return Err(diagnostic.reason("replay"));
|
||||
}
|
||||
@@ -247,29 +281,58 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
runtime_id: claims.iss.clone(),
|
||||
worker_id: worker_id.to_owned(),
|
||||
};
|
||||
let member = store
|
||||
.get_worker_registry(workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("worker_catalog_authority_unavailable")
|
||||
})?;
|
||||
let reserved = store
|
||||
.has_active_worker_create_reservation(workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("worker_reservation_authority_unavailable")
|
||||
})?;
|
||||
if member.is_none() && !reserved {
|
||||
return Err(diagnostic.reason("worker_catalog_membership"));
|
||||
}
|
||||
store
|
||||
.require_current_worker_singleton_owner(workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
diagnostic
|
||||
.clone()
|
||||
.reason("worker_singleton_authority_denied")
|
||||
let workspace_id = workspace_id.to_owned();
|
||||
let executor = store.db_executor();
|
||||
let stage_diagnostic = diagnostic.clone();
|
||||
// Only synchronous catalog authority lives in this closure. The trust
|
||||
// and JTI async wrappers above already offload through the executor.
|
||||
executor
|
||||
.run(move || {
|
||||
let member = store
|
||||
.get_worker_registry(&workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.unavailable("worker_catalog_authority_unavailable")
|
||||
})?;
|
||||
let reserved = store
|
||||
.has_active_worker_create_reservation(&workspace_id, &worker)
|
||||
.map_err(|_| {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.unavailable("worker_reservation_authority_unavailable")
|
||||
})?;
|
||||
if member.is_none() && !reserved {
|
||||
return Err(stage_diagnostic.reason("worker_catalog_membership"));
|
||||
}
|
||||
store
|
||||
.require_current_worker_singleton_owner(&workspace_id, &worker)
|
||||
.map_err(|error| {
|
||||
if matches!(
|
||||
error,
|
||||
crate::Error::DbCapacityExceeded
|
||||
| crate::Error::DbExecutorUnavailable
|
||||
| crate::Error::DbOperationOutcomeUnknown
|
||||
| crate::Error::Sqlite(_)
|
||||
) {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.unavailable("worker_singleton_authority_unavailable")
|
||||
} else {
|
||||
stage_diagnostic
|
||||
.clone()
|
||||
.reason("worker_singleton_authority_denied")
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|error: RuntimeRequestProofError| {
|
||||
if error.verification == "unverified" && error.is_availability_error() {
|
||||
diagnostic.clone().unavailable(error.reason)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
Ok(VerifiedRuntimeRequestSource {
|
||||
@@ -279,3 +342,290 @@ pub(crate) async fn verify_runtime_request_source_proof_with_clock(
|
||||
authentication_binding_id: trusted.binding_id,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::{
|
||||
AccountRecord, SqliteWorkspaceStore, WorkspaceRecord, WorkspaceRuntimeAuthenticationMode,
|
||||
WorkspaceRuntimeBinding,
|
||||
};
|
||||
use std::time::Duration;
|
||||
use worker_runtime::auth::{
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial, RuntimeRequestSourceSigner,
|
||||
request_body_digest,
|
||||
};
|
||||
|
||||
const WORKSPACE: &str = "019d0000-0000-7000-8000-0000000000aa";
|
||||
const NOW: i64 = 1_700_000_000;
|
||||
const PATH: &str = "/api/runtime/v1/resources/fetch";
|
||||
|
||||
// rusqlite's busy handler accepts a function pointer, not a closure. Each
|
||||
// fixture owns a dedicated single-worker executor, so this thread-local
|
||||
// handshake is isolated from all parallel tests and product connections.
|
||||
thread_local! {
|
||||
static SQLITE_BUSY_HANDSHAKE: std::cell::RefCell<Option<(
|
||||
tokio::sync::oneshot::Sender<()>,
|
||||
std::sync::mpsc::Receiver<()>,
|
||||
)>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn acknowledge_sqlite_contention(_attempt: i32) -> bool {
|
||||
SQLITE_BUSY_HANDSHAKE.with(|slot| {
|
||||
let Some((blocked, retry)) = slot.borrow_mut().take() else {
|
||||
return false;
|
||||
};
|
||||
let _ = blocked.send(());
|
||||
// The test releases SQLite's writer lock before allowing retry.
|
||||
// Dropping the sender on panic also releases this worker promptly.
|
||||
retry.recv_timeout(Duration::from_secs(3)).is_ok()
|
||||
})
|
||||
}
|
||||
|
||||
async fn fixture(
|
||||
root: &std::path::Path,
|
||||
) -> (Arc<SqliteWorkspaceStore>, ServerConfig, String, String) {
|
||||
let db_path = root.join("auth.db");
|
||||
let executor = crate::db_executor::DbExecutor::new(crate::db_executor::DbExecutorLimits {
|
||||
workers: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
queued: std::num::NonZeroUsize::new(1).unwrap(),
|
||||
})
|
||||
.unwrap();
|
||||
let store = Arc::new(
|
||||
SqliteWorkspaceStore::open_for_test(&db_path)
|
||||
.unwrap()
|
||||
.with_db_executor(executor),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-auth".into(),
|
||||
kind: "user".into(),
|
||||
handle: "auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
let workspace = WorkspaceRecord {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
owner_account_id: "account-auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
};
|
||||
store.upsert_workspace(&workspace).await.unwrap();
|
||||
let config = ServerConfig {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
workspace_display_name: "Auth".into(),
|
||||
workspace_created_at: "1".into(),
|
||||
workspace_execution_root: root.to_owned(),
|
||||
database_path: db_path,
|
||||
data_root: root.to_owned(),
|
||||
embedded_runtime_store_root: root.join("runtime"),
|
||||
static_assets_dir: None,
|
||||
auth: crate::server::AuthConfig::Passkey {
|
||||
rp_id: "localhost".into(),
|
||||
origin: "http://localhost".into(),
|
||||
public_base_url: "http://localhost".into(),
|
||||
cookie_name: "session".into(),
|
||||
},
|
||||
max_records: 10,
|
||||
repositories: vec![],
|
||||
runtime_event_sources: vec![],
|
||||
remote_runtime_sources: vec![],
|
||||
backend_base_url: Some("server-auth".into()),
|
||||
};
|
||||
let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap();
|
||||
store
|
||||
.upsert_workspace_runtime_binding_record(
|
||||
WorkspaceRuntimeBinding {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
runtime_id: "runtime-auth".into(),
|
||||
display_name: "Auth".into(),
|
||||
base_url: "http://127.0.0.1:1".into(),
|
||||
public_key: identity.public_key.clone(),
|
||||
public_key_fingerprint: String::new(),
|
||||
binding_id: String::new(),
|
||||
authentication_mode: WorkspaceRuntimeAuthenticationMode::LegacyServerIssuer,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
revoked_at: None,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let binding = ControlPlaneStore::get_workspace_runtime_binding(
|
||||
store.as_ref(),
|
||||
WORKSPACE,
|
||||
"runtime-auth",
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let proof = RuntimeRequestSourceSigner::from_identity(&identity)
|
||||
.issue(
|
||||
"server-auth",
|
||||
WORKSPACE,
|
||||
None,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
b"{}",
|
||||
NOW,
|
||||
30,
|
||||
)
|
||||
.unwrap();
|
||||
(store, config, proof, binding.binding_id)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn sqlite_blocked_jti_keeps_timer_and_control_tasks_responsive_and_rejects_replay() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (store, config, proof, binding_id) = fixture(root.path()).await;
|
||||
// A real independent SQLite writer prevents JTI insertion while trust
|
||||
// reads remain possible. Rollback on Drop also releases this on panic.
|
||||
let mut writer = rusqlite::Connection::open(&config.database_path).unwrap();
|
||||
let lock = writer
|
||||
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let (blocked, contention_reached) = tokio::sync::oneshot::channel();
|
||||
let (retry, retry_rx) = std::sync::mpsc::channel();
|
||||
let setup_store = store.clone();
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || -> crate::Result<()> {
|
||||
SQLITE_BUSY_HANDSHAKE.with(|slot| *slot.borrow_mut() = Some((blocked, retry_rx)));
|
||||
setup_store.with_conn(|conn| {
|
||||
conn.busy_handler(Some(acknowledge_sqlite_contention))?;
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let request_store = store.clone();
|
||||
let request_config = config.clone();
|
||||
let request_proof = proof.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
verify_runtime_request_source_proof_with_clock(
|
||||
request_store,
|
||||
&request_config,
|
||||
&request_proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), contention_reached)
|
||||
.await
|
||||
.expect(
|
||||
"SQLite JTI persistence must report real writer contention without blocking Tokio",
|
||||
)
|
||||
.unwrap();
|
||||
// The busy callback acknowledges an actual SQLite write blocked by the
|
||||
// independent writer, then holds retry until explicitly released.
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"JTI must remain pending while SQLite writer is locked"
|
||||
);
|
||||
let control = tokio::spawn(async { "control-responsive" });
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), control)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
"control-responsive"
|
||||
);
|
||||
// This timer is the behavior being tested, not a guessed wait for SQL.
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
tokio::time::sleep(Duration::from_millis(1)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"timer must advance while JTI remains blocked"
|
||||
);
|
||||
lock.rollback().unwrap();
|
||||
retry.send(()).unwrap();
|
||||
let source = tokio::time::timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(source.runtime_id, "runtime-auth");
|
||||
assert_eq!(source.authentication_binding_id, binding_id);
|
||||
let replay = verify_runtime_request_source_proof_with_clock(
|
||||
store,
|
||||
&config,
|
||||
&proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(replay.reason, "replay");
|
||||
assert!(!replay.is_availability_error());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_database_is_availability_failure_not_invalid_proof() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (store, config, proof, _) = fixture(root.path()).await;
|
||||
// Fill the isolated executor with a blocked job and queued jobs. No
|
||||
// scheduling sleeps: the worker explicitly acknowledges its start.
|
||||
let executor = store.db_executor();
|
||||
let (started, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker_executor = executor.clone();
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_executor
|
||||
.run(move || -> crate::Result<()> {
|
||||
let _ = started.send(());
|
||||
release_rx.recv().unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut queued = Box::pin(executor.run(|| -> crate::Result<()> { Ok(()) }));
|
||||
assert!(futures::poll!(queued.as_mut()).is_pending());
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
verify_runtime_request_source_proof_with_clock(
|
||||
store,
|
||||
&config,
|
||||
&proof,
|
||||
WORKSPACE,
|
||||
BACKEND_RESOURCE_FETCH_PERMISSION,
|
||||
"POST",
|
||||
PATH,
|
||||
&request_body_digest(b"{}"),
|
||||
&|| NOW,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
// Release before assertions so a failed contract never strands a worker.
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap().unwrap();
|
||||
queued.await.unwrap();
|
||||
let error = outcome
|
||||
.expect("capacity rejection must not wait for queued SQL")
|
||||
.unwrap_err();
|
||||
assert_eq!(error.reason, "runtime_trust_authority_unavailable");
|
||||
assert!(error.is_availability_error());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_is_atomic_and_exact_retries_converge() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
Arc::new(
|
||||
@@ -339,7 +339,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn create_rejects_noncanonical_repository_keys_without_persisting_a_workspace() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
Arc::new(
|
||||
@@ -378,7 +378,7 @@ mod tests {
|
||||
for with_repository in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let database_path = temp.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
@@ -418,7 +418,7 @@ mod tests {
|
||||
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let restarted = WorkspaceCatalogService::new(store.clone(), materials);
|
||||
let created = restarted.create(request, owner_account_id).unwrap();
|
||||
let identity = store
|
||||
@@ -435,7 +435,7 @@ mod tests {
|
||||
for with_repository in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let database_path = temp.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let materials = Arc::new(InMemoryWorkspaceSigningMaterialStore::default());
|
||||
let service = WorkspaceCatalogService::new(store.clone(), materials.clone());
|
||||
@@ -486,7 +486,7 @@ mod tests {
|
||||
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let restarted = WorkspaceCatalogService::new(store.clone(), materials);
|
||||
let created = restarted.create(request, owner_account_id).unwrap();
|
||||
let identity = store
|
||||
@@ -500,7 +500,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn idempotency_key_reuse_with_different_payload_is_rejected() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store,
|
||||
@@ -532,7 +532,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn repository_free_create_initializes_authority_and_concurrent_retries_converge() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
@@ -611,7 +611,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replay_reads_current_repository_at_initial_key_and_never_recreates_it() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
@@ -673,13 +673,13 @@ mod tests {
|
||||
repository: None,
|
||||
};
|
||||
let (owner, created) = {
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
|
||||
let owner = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(store, materials.clone());
|
||||
let created = service.create(request.clone(), owner.clone()).unwrap();
|
||||
(owner, created)
|
||||
};
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database).unwrap());
|
||||
let reopened = WorkspaceCatalogService::new(store.clone(), materials);
|
||||
let replay = reopened.create(request, owner).unwrap();
|
||||
assert_eq!(replay.workspace, created.workspace);
|
||||
@@ -739,7 +739,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn create_rejects_non_user_account_owners() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "organization-owner".to_string(),
|
||||
@@ -777,7 +777,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn catalog_list_is_scoped_to_the_required_owner_account() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner_a = owner_account(store.as_ref());
|
||||
let owner_b = "account-owner-b".to_string();
|
||||
store
|
||||
@@ -845,7 +845,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn remote_repository_creation_persists_typed_source_without_auth_metadata() {
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::in_memory_for_test().unwrap());
|
||||
let owner_account_id = owner_account(store.as_ref());
|
||||
let service = WorkspaceCatalogService::new(
|
||||
store.clone(),
|
||||
|
||||
@@ -236,7 +236,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
"Workspace deletion request exceeds bounded field limits".to_string(),
|
||||
));
|
||||
}
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
if let Some(existing) = read_operation(tx, &request.operation_id)? {
|
||||
if existing.actor_account_id != actor_account_id {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
@@ -369,7 +369,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
child_operation_id: &str,
|
||||
) -> Result<WorkspaceDeletionOperationResponse> {
|
||||
validate_operation_id(child_operation_id)?;
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
let operation = read_operation(tx, operation_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?
|
||||
.response;
|
||||
@@ -411,7 +411,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
failure_category: Option<&str>,
|
||||
) -> Result<WorkspaceDeletionOperationResponse> {
|
||||
validate_operation_projection(child_operation_ids, blockers)?;
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
let current = read_operation(tx, operation_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?
|
||||
.response;
|
||||
@@ -462,7 +462,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
&self,
|
||||
operation_id: &str,
|
||||
) -> Result<WorkspaceDeletionOperationResponse> {
|
||||
self.with_transaction(|tx| {
|
||||
self.with_immediate_transaction(|tx| {
|
||||
tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?;
|
||||
let operation = read_operation(tx, operation_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?;
|
||||
@@ -922,7 +922,7 @@ mod tests {
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn setup() -> (SqliteWorkspaceStore, String, String) {
|
||||
seed_workspaces(SqliteWorkspaceStore::in_memory().expect("store"))
|
||||
seed_workspaces(SqliteWorkspaceStore::in_memory_for_test().expect("store"))
|
||||
}
|
||||
|
||||
fn seed_workspaces(store: SqliteWorkspaceStore) -> (SqliteWorkspaceStore, String, String) {
|
||||
@@ -989,7 +989,7 @@ mod tests {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let path = dir.path().join("server.db");
|
||||
let (store, owner, workspace_id) =
|
||||
seed_workspaces(SqliteWorkspaceStore::open(&path).expect("store"));
|
||||
seed_workspaces(SqliteWorkspaceStore::open_for_test(&path).expect("store"));
|
||||
for workspace in ["workspace-a", "workspace-b"] {
|
||||
let backend = ticket::SqliteTicketBackend::open_verified(&path, workspace).unwrap();
|
||||
let initial = backend
|
||||
@@ -1019,7 +1019,7 @@ mod tests {
|
||||
// Seed domain handling evidence under a transaction. This fixture
|
||||
// exercises owner purge, not singleton authentication or delivery.
|
||||
store
|
||||
.with_transaction(|tx| {
|
||||
.with_immediate_transaction(|tx| {
|
||||
ticket::sqlite_update_ticket_assignment_tracking_in_transaction(
|
||||
tx,
|
||||
workspace,
|
||||
|
||||
@@ -58,10 +58,13 @@ pub(crate) struct WorkspaceDriveHost {
|
||||
impl WorkspaceDriveHost {
|
||||
pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result<Self> {
|
||||
let storage = storage.clone();
|
||||
let registration = tokio::task::spawn_blocking(move || Drive::register(&storage))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))?
|
||||
.map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?;
|
||||
let registration = crate::db_executor::DbExecutor::shared()
|
||||
.run(move || {
|
||||
Drive::register(&storage).map_err(|error| {
|
||||
Error::Store(format!("failed to register Drive storage: {error}"))
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
Ok(Self {
|
||||
registration,
|
||||
purged: AtomicBool::new(false),
|
||||
@@ -80,21 +83,21 @@ impl WorkspaceDriveHost {
|
||||
}
|
||||
let storage = storage.clone();
|
||||
let registration = self.registration.clone();
|
||||
let drive = tokio::task::spawn_blocking(move || {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain IMMEDIATE transactions lock the attached server authority
|
||||
// as well as Drive metadata. Deletion reservation and cached-handle
|
||||
// operations therefore serialize with other server DB writes.
|
||||
Drive::open_with_workspace_authority(
|
||||
&storage,
|
||||
®istration,
|
||||
&root,
|
||||
&authority_database,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??;
|
||||
let drive = crate::db_executor::DbExecutor::shared()
|
||||
.run(move || {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain IMMEDIATE transactions lock the attached server authority
|
||||
// as well as Drive metadata. Deletion reservation and cached-handle
|
||||
// operations therefore serialize with other server DB writes.
|
||||
Drive::open_with_workspace_authority(
|
||||
&storage,
|
||||
®istration,
|
||||
&root,
|
||||
&authority_database,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await?;
|
||||
*cached = Some(drive.clone());
|
||||
Ok(drive)
|
||||
}
|
||||
@@ -109,9 +112,9 @@ impl WorkspaceDriveHost {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root, authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from))
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || drive.fence().map_err(Error::from))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))?
|
||||
}
|
||||
|
||||
pub(crate) async fn purge(
|
||||
@@ -124,20 +127,20 @@ impl WorkspaceDriveHost {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root.clone(), authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
|
||||
drive.purge()?;
|
||||
// Remove the empty blob root, never the data root or another Workspace.
|
||||
// Missing roots are successful retries; other failures retain FeatureStorage.
|
||||
match std::fs::remove_dir_all(&root) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??;
|
||||
crate::db_executor::DbExecutor::shared()
|
||||
.run(move || -> Result<()> {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
|
||||
drive.purge()?;
|
||||
// Remove the empty blob root, never the data root or another Workspace.
|
||||
// Missing roots are successful retries; other failures retain FeatureStorage.
|
||||
match std::fs::remove_dir_all(&root) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
// A later signing-material/finalization failure can retry after the
|
||||
// feature DB has been closed and removed by FeatureStorage.delete.
|
||||
self.purged.store(true, Ordering::Release);
|
||||
|
||||
@@ -780,7 +780,7 @@ mod tests {
|
||||
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let database_path = temp.path().join("server.db");
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "account-1".to_string(),
|
||||
@@ -833,7 +833,7 @@ mod tests {
|
||||
|
||||
drop(failing_service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let service = WorkspaceSigningIdentityService::new(store.clone(), materials.clone());
|
||||
let provisioned = service
|
||||
.provision_existing("workspace-1", "account-1")
|
||||
@@ -962,7 +962,7 @@ mod tests {
|
||||
);
|
||||
drop(service);
|
||||
drop(store);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open(&database_path).unwrap());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&database_path).unwrap());
|
||||
let restarted = WorkspaceSigningIdentityService::new(store.clone(), materials.clone());
|
||||
let recovered = restarted
|
||||
.provision_existing("workspace-2", "account-1")
|
||||
|
||||
@@ -117,11 +117,13 @@ pub(crate) async fn serve_workspace_subscription<S>(
|
||||
worker_id,
|
||||
runtime_id: Some(runtime_id),
|
||||
} => {
|
||||
let connection = match WorkspaceWorker::resolve(
|
||||
let connection = match WorkspaceWorker::resolve_async(
|
||||
&api,
|
||||
&runtime_id,
|
||||
worker_id.as_str(),
|
||||
) {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(worker) => worker
|
||||
.connect_protocol(&context)
|
||||
.await
|
||||
@@ -366,7 +368,8 @@ async fn run_workspace_workers(
|
||||
) {
|
||||
let Ok(mut projection) = api
|
||||
.worker_projection
|
||||
.subscribe_ordered(api.config.max_records.max(1))
|
||||
.subscribe_ordered_async(api.config.max_records.max(1))
|
||||
.await
|
||||
else {
|
||||
let _ = send_rejected(
|
||||
&outbound,
|
||||
@@ -377,11 +380,22 @@ async fn run_workspace_workers(
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let mut workers = projection
|
||||
.take_snapshot()
|
||||
.into_iter()
|
||||
.filter_map(|record| project_registry_worker(&api, record))
|
||||
.collect::<Vec<_>>();
|
||||
let Ok(mut workers) = project_registry_workers_async(
|
||||
api.store.clone(),
|
||||
api.config.workspace_id.clone(),
|
||||
projection.take_snapshot(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
let _ = send_rejected(
|
||||
&outbound,
|
||||
request_id,
|
||||
SubscriptionRejectionCode::Internal,
|
||||
"failed to enrich durable Worker projection".to_string(),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
sort_workers(&mut workers);
|
||||
if send_frame(
|
||||
&outbound,
|
||||
@@ -406,7 +420,29 @@ async fn run_workspace_workers(
|
||||
for change in event.changes {
|
||||
let payload = match change {
|
||||
crate::worker_projection::WorkerProjectionChange::Upsert(record) => {
|
||||
let Some(worker) = project_registry_worker(&api, record) else {
|
||||
let workers = project_registry_workers_async(
|
||||
api.store.clone(),
|
||||
api.config.workspace_id.clone(),
|
||||
vec![record],
|
||||
)
|
||||
.await;
|
||||
let Ok(mut workers) = workers else {
|
||||
// Admission failure cannot silently lose a live
|
||||
// update: terminate and require a fresh snapshot.
|
||||
let _ = send_frame(
|
||||
&outbound,
|
||||
SubscriptionFrame::new(SubscriptionFramePayload::Event(
|
||||
SubscriptionEvent::SubscriptionClosed {
|
||||
subscription_id: subscription_id.clone(),
|
||||
code: SubscriptionTerminationCode::Lagged,
|
||||
message: "Worker projection enrichment unavailable; reconnect for a fresh snapshot".to_string(),
|
||||
},
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let Some(worker) = workers.pop() else {
|
||||
continue;
|
||||
};
|
||||
SubscriptionEventPayload::WorkerUpserted { worker }
|
||||
@@ -450,8 +486,28 @@ async fn run_workspace_workers(
|
||||
}
|
||||
}
|
||||
|
||||
async fn project_registry_workers_async(
|
||||
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
|
||||
workspace_id: String,
|
||||
records: Vec<crate::store::WorkerRegistryProjectionRecord>,
|
||||
) -> crate::Result<Vec<SubscriptionWorker>> {
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
// Keep the committed Worker payload; only the existing Workspace-scoped
|
||||
// workdir/history/repository enrichment reads occur inside this job.
|
||||
Ok(records
|
||||
.into_iter()
|
||||
.filter_map(|record| project_registry_worker(store.as_ref(), &workspace_id, record))
|
||||
.collect())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
// Synchronous helpers stay synchronous; they run only inside the job above.
|
||||
fn project_registry_worker(
|
||||
api: &WorkspaceApi,
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
record: crate::store::WorkerRegistryProjectionRecord,
|
||||
) -> Option<SubscriptionWorker> {
|
||||
let runtime_id = record.registry.worker.runtime_id.clone();
|
||||
@@ -491,29 +547,25 @@ fn project_registry_worker(
|
||||
attempt_id: job.attempt_id,
|
||||
purpose: job.purpose,
|
||||
});
|
||||
if !project_working_directory(api, &record.registry.worker, &mut worker) {
|
||||
if !project_working_directory(store, workspace_id, &record.registry.worker, &mut worker) {
|
||||
return None;
|
||||
}
|
||||
Some(worker)
|
||||
}
|
||||
|
||||
fn project_working_directory(
|
||||
api: &WorkspaceApi,
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
worker_ref: &RuntimeWorkerRef,
|
||||
worker: &mut SubscriptionWorker,
|
||||
) -> bool {
|
||||
let Ok(links) = api
|
||||
.store
|
||||
.list_worker_workdir_links(&api.config.workspace_id, worker_ref)
|
||||
else {
|
||||
let Ok(links) = store.list_worker_workdir_links(workspace_id, worker_ref) else {
|
||||
return false;
|
||||
};
|
||||
if !links.is_empty() {
|
||||
let mut attachments = Vec::with_capacity(links.len());
|
||||
for link in links {
|
||||
let Ok(Some(workdir)) = api
|
||||
.store
|
||||
.get_workdir_registry(&api.config.workspace_id, &link.workdir_id)
|
||||
let Ok(Some(workdir)) = store.get_workdir_registry(workspace_id, &link.workdir_id)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -533,9 +585,8 @@ fn project_working_directory(
|
||||
}
|
||||
worker.workdir_attachments = attachments;
|
||||
} else {
|
||||
let Ok(has_link_history) = api
|
||||
.store
|
||||
.worker_workdir_link_history_exists(&api.config.workspace_id, worker_ref)
|
||||
let Ok(has_link_history) =
|
||||
store.worker_workdir_link_history_exists(workspace_id, worker_ref)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -543,13 +594,17 @@ fn project_working_directory(
|
||||
worker.workdir_attachments.clear();
|
||||
}
|
||||
}
|
||||
project_repository_keys(api, worker)
|
||||
project_repository_keys(store, workspace_id, worker)
|
||||
}
|
||||
|
||||
fn project_repository_keys(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool {
|
||||
fn project_repository_keys(
|
||||
store: &dyn crate::store::ControlPlaneStore,
|
||||
workspace_id: &str,
|
||||
worker: &mut SubscriptionWorker,
|
||||
) -> bool {
|
||||
project_repository_keys_with(worker, |repository_id| {
|
||||
api.store
|
||||
.get_repository(&api.config.workspace_id, repository_id)
|
||||
store
|
||||
.get_repository(workspace_id, repository_id)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|repository| repository.repository_key)
|
||||
@@ -634,9 +689,8 @@ mod tests {
|
||||
protocol::subscription::SubscriptionWorkerId::new("worker-1").unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||
let mut worker = SubscriptionWorker {
|
||||
fn external_worker() -> SubscriptionWorker {
|
||||
SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: worker_id(),
|
||||
runtime_id: Some("runtime-1".to_string()),
|
||||
@@ -660,8 +714,12 @@ mod tests {
|
||||
.unwrap(),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||
let mut worker = external_worker();
|
||||
assert!(project_repository_keys_with(&mut worker, |_| {
|
||||
panic!("External Workdir must not require Repository projection")
|
||||
}));
|
||||
@@ -673,6 +731,110 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn async_enrichment_preserves_committed_worker_and_external_attachment_but_hides_unresolved_repository()
|
||||
{
|
||||
use crate::store::{
|
||||
AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerRegistryRecord,
|
||||
WorkspaceRecord,
|
||||
};
|
||||
let store = std::sync::Arc::new(
|
||||
SqliteWorkspaceStore::in_memory_for_test()
|
||||
.unwrap()
|
||||
.with_db_executor(crate::db_executor::DbExecutor::new(Default::default()).unwrap()),
|
||||
);
|
||||
store
|
||||
.upsert_account(&AccountRecord {
|
||||
account_id: "owner".into(),
|
||||
kind: "user".into(),
|
||||
handle: "owner".into(),
|
||||
display_name: "Owner".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-1".into(),
|
||||
owner_account_id: "owner".into(),
|
||||
display_name: "Enrichment".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let worker_ref = RuntimeWorkerRef::new("runtime-1", "worker-1");
|
||||
store
|
||||
.upsert_worker_registry(&WorkerRegistryRecord {
|
||||
workspace_id: "workspace-1".into(),
|
||||
worker: worker_ref.clone(),
|
||||
display_name: "Durable name".into(),
|
||||
profile: Some("builtin:coder".into()),
|
||||
retention_state: "normal".into(),
|
||||
transcript_ref: None,
|
||||
session_ref: None,
|
||||
summary_ref: None,
|
||||
diagnostics_ref: None,
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.apply_worker_registry_observation("workspace-1", "runtime-1", &external_worker(), "2")
|
||||
.unwrap();
|
||||
let committed = store
|
||||
.worker_registry_projection("workspace-1", &worker_ref)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
// Change authority after capturing the payload, as can happen while async
|
||||
// enrichment is queued. It must not re-query and replace the committed Worker.
|
||||
let mut later = external_worker();
|
||||
later.state = protocol::subscription::SubscriptionWorkerState::Running;
|
||||
store
|
||||
.apply_worker_registry_observation("workspace-1", "runtime-1", &later, "3")
|
||||
.unwrap();
|
||||
let projected = project_registry_workers_async(
|
||||
store.clone(),
|
||||
"workspace-1".into(),
|
||||
vec![committed.clone()],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(projected.len(), 1);
|
||||
let worker = &projected[0];
|
||||
assert_eq!(
|
||||
worker.state,
|
||||
protocol::subscription::SubscriptionWorkerState::Idle
|
||||
);
|
||||
assert_eq!(
|
||||
worker.availability,
|
||||
protocol::subscription::SubscriptionWorkerAvailability::Observed
|
||||
);
|
||||
assert_eq!(worker.display_name.as_deref(), Some("Durable name"));
|
||||
assert_eq!(worker.profile.as_deref(), Some("builtin:coder"));
|
||||
assert_eq!(worker.runtime_id.as_deref(), Some("runtime-1"));
|
||||
assert_eq!(worker.workspace_id.as_deref(), Some("workspace-1"));
|
||||
assert_eq!(
|
||||
worker.workdir_attachments,
|
||||
external_worker().workdir_attachments
|
||||
);
|
||||
let mut unresolved = committed;
|
||||
unresolved
|
||||
.observation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.worker
|
||||
.workdir_attachments[0]
|
||||
.repository_id = Some("missing-repository".into());
|
||||
assert!(
|
||||
project_registry_workers_async(store, "workspace-1".into(), vec![unresolved])
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_with_events(events: mpsc::Receiver<protocol::Event>) -> SubscriptionResponse {
|
||||
let (control_outbound, mut control_receiver) = mpsc::channel(1);
|
||||
let (protocol_outbound, _protocol_receiver) = mpsc::channel(1);
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
# Backend bounded synchronous execution
|
||||
|
||||
## Ownership and admission
|
||||
|
||||
`ControlPlaneStore::db_executor()` returns a clone of the bounded execution owner.
|
||||
Use `executor.run(move || synchronous_operation(...)).await` with owned input and
|
||||
services. The closure owns the **whole** synchronous operation, including connection
|
||||
acquisition, mutex wait, the existing SQL transaction, commit, and ordered local
|
||||
publication. Do not split source authorization, receipts, CAS, audit, or saved
|
||||
results around an await. Do not change connection ownership or introduce a read pool.
|
||||
|
||||
Production stores share one DB worker and a queue of 32 jobs. Admission uses a
|
||||
nonblocking queue send and a separate actual-job budget (worker count plus queue
|
||||
capacity); rejection means the operation was **not accepted**. HTTP maps capacity
|
||||
and executor unavailability to 503, not authentication failure or deletion conflict.
|
||||
Tests can inject a dedicated executor with explicit limits, without process-global
|
||||
settings or environment variables. Ordinary SQLite fixtures use `open_for_test`,
|
||||
`in_memory_for_test`, or `from_connection_for_test`: their clones share their own
|
||||
1-worker/32-queued owner, not the production singleton or another fixture's drain.
|
||||
Server/Workspace fixture constructors likewise inject a lazy `HostExecutors` owner;
|
||||
all child Workspaces and weak retention adapters use that same owner. Host and
|
||||
cleanup still have the production 2-worker/16-queued limits. Dropping the final
|
||||
sender closes its queue after accepted work finishes; idle fixture threads are
|
||||
not kept in a process-global fixture cache.
|
||||
|
||||
The restore intent journal and Workspace deletion use
|
||||
`with_immediate_transaction`: read authority/journal state only after acquiring
|
||||
SQLite write ownership. A DEFERRED read-to-write upgrade can return BUSY without
|
||||
invoking SQLite's busy handler when another connection is writing. Moving these
|
||||
whole helpers to an offloaded caller must not create an automatic mutation retry.
|
||||
|
||||
|
||||
Accepted work is owned by the worker, not the Tokio waiter. Cancelling a waiter:
|
||||
|
||||
- does not interrupt, resubmit, or retry SQL;
|
||||
- does not return its queue/job slot before execution and discarded-result cleanup;
|
||||
- does not release the request lifecycle lease before actual completion.
|
||||
|
||||
Synchronous post-commit continuations inherit the admitted lifecycle source when
|
||||
submitting another bounded job. This is not implicit inheritance by detached Tokio
|
||||
tasks. An operation panic has an unknown outcome even if SQL already committed;
|
||||
return a structured unknown-outcome error and diagnostic, never retry the mutation.
|
||||
A cancelled fire-and-forget job still emits its diagnostic. `drain()` observes all
|
||||
jobs accepted before its barrier is first polled and waits for actual cleanup; it
|
||||
is not a queue close and does not include later admissions. Lifecycle callers must
|
||||
stop their producers/admission before destroying resources.
|
||||
|
||||
## Runtime/network boundaries
|
||||
|
||||
A synchronous Runtime workflow must not monopolize the sole DB worker while
|
||||
waiting for callbacks that themselves need DB authority. The Server's separate
|
||||
finite Host lane has two workers and 16 queued jobs. It uses the same accepted-job
|
||||
ownership mechanism, not an unbounded `spawn_blocking` queue.
|
||||
|
||||
Pure DB stages use the DB lane. Existing mixed resource-lifecycle workflows can
|
||||
use the Host lane, with no Runtime I/O or await inside a SQL transaction. Where an
|
||||
owned workflow needs async resource locks, `run_host_workflow` polls it on a Host
|
||||
worker using the original Tokio handle; do not build a replacement runtime for
|
||||
cached services and do not submit to and await the Host lane from inside itself.
|
||||
Control/session guards needed by an accepted mutation belong to that actual job.
|
||||
Subjektiv session Read/Search, subject response projection, general memory staging
|
||||
and legacy consolidation-scope helpers also use Host ownership: they can
|
||||
synchronously call Runtime/network services outside their SQL transactions.
|
||||
Pure DB-resident Subjektiv scope and memory projection helpers stay on the DB lane.
|
||||
Host admission must not be acquired again from an already-owned Host workflow,
|
||||
including indirectly through a Runtime's mandatory Backend callback.
|
||||
|
||||
Mandatory Session attribution has Subject-only authority and whole attribution
|
||||
SQL on the independent DB lane. Store-only scope discovery distinguishes Subject,
|
||||
current Job consolidation, and legacy consolidation authority; attribution rejects
|
||||
both consolidation roles **before** any Runtime profile/tag lookup. Legacy
|
||||
consolidation checks for mixed callers remain on Host. Subject-only memory
|
||||
operations (resident context/summary, proposal validation, receipt lookup, explicit
|
||||
staging) use the same store-only authority boundary and DB ownership. Runtime
|
||||
bootstrap/restore can request resident context while its Host parent is waiting.
|
||||
This is not permission to bypass Workspace deletion admission: only the already
|
||||
allowed exact callback endpoints may reenter deletion drain.
|
||||
|
||||
Backend Job cleanup has a third finite lane (two workers, 16 queued jobs), so a
|
||||
long Run teardown wait cannot starve ordinary Host control/stop requests. It is
|
||||
admitted before its durable cleanup claim. Rejection leaves Pending/Failed
|
||||
ownership recoverable instead of abandoning an Executing claim. Ticket
|
||||
checker dispatch and timeout processing use bounded admission; overload leaves the
|
||||
saved reservation/deadline diagnosable. They are not automatically retried after an
|
||||
unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof
|
||||
of model processing.
|
||||
|
||||
Remote subscription and Workdir authorization are async callbacks: their complete
|
||||
live binding check and fresh request signing are offloaded together. Cold protocol
|
||||
source lookup in `connect_workspace_worker_protocol` is also awaited as an owned
|
||||
DB stage, rather than invoking its synchronous signing callback on Tokio.
|
||||
`WorkspaceWorkerRuntime::observation_source` is fallible: live binding/signing
|
||||
failure and bounded admission errors propagate explicitly, not as a missing
|
||||
source or a fallback to a configured bearer token. Synchronous CLI/migration
|
||||
callers and existing synchronous Runtime adapters retain their explicit blocking
|
||||
boundary.
|
||||
|
||||
## Caller recipe (including assignment APIs)
|
||||
|
||||
```rust
|
||||
let store = api.store.clone();
|
||||
let workspace_id = api.workspace_id().to_owned();
|
||||
let worker = worker.clone();
|
||||
let record = api.store.db_executor().run(move || {
|
||||
store.get_worker_registry(&workspace_id, &worker)
|
||||
}).await?;
|
||||
```
|
||||
|
||||
This is an illustration, not a new authority API. Assignment delivery scan,
|
||||
reserve, prepare, acknowledgement and start authorization must keep their existing
|
||||
whole transaction helpers. Additional assignment diagnostics, answer mutations,
|
||||
and Ticket queries use the same owned caller boundary; do not add a second ledger
|
||||
or reconstruct processing evidence from Worker liveness.
|
||||
|
||||
Worker discovery uses the same whole `workers_response` DB stage as Worker list
|
||||
and resource detail: projection snapshot, Workdir registry/links, and authority
|
||||
projection never acquire the shared connection on Tokio. Source-proof requirement
|
||||
and pure query/cursor validation still precede admission; overload remains 503.
|
||||
Runtime resource list/detail use an accepted Host workflow for synchronous Runtime
|
||||
summary I/O and await independent DB binding/audit reads. Launch options are also
|
||||
mixed Runtime/config helpers and use Host, not the sole DB worker. Runtime
|
||||
registration validates its signing identity in a whole DB stage, scans existing
|
||||
Workers on Host, then owns SQL key/CAS mutation, local registration publication
|
||||
and response observation in one accepted workflow. Metadata updates likewise keep
|
||||
commit and publication together after waiter cancellation. Trust-key revoke also
|
||||
owns whole CAS/audit SQL, expectation removal, broker unregister/Shutdown and
|
||||
response observation in one accepted Host workflow. Future request proof denial
|
||||
alone does not terminate existing subscriptions. Host admission must precede the
|
||||
mutation, so overload cannot commit a revoke then fail on response admission.
|
||||
Public resource/detail helpers admit Host ownership; already-owned workflows call their `_on_host`
|
||||
variants directly, never await another admission to the same occupied lane.
|
||||
Connection tests ping on finite Host ownership and then await the whole current
|
||||
binding fence on DB. A replaced/revoked binding cannot attest the earlier ping.
|
||||
Do not confuse a Runtime summary callback with a pure stored Worker projection
|
||||
or run network waits on the sole DB worker.
|
||||
|
||||
## Regression evidence
|
||||
|
||||
- `db_executor::tests`: real SQLite IMMEDIATE write contention while a current-thread
|
||||
timer/control task progresses; accepted/cancelled queue bounds; rollback; panic
|
||||
before/after commit; discarded reply cleanup and retained authority; child jobs.
|
||||
- `server::db_execution_tests`: HTTP Ticket writes and cold/warm Subjektiv scope
|
||||
opening/mutation under a separate SQLite connection's write lock; overload;
|
||||
cancelled HTTP jobs and Workspace deletion drain; same-budget fixture isolation,
|
||||
saturated/cancelled Host jobs versus cleanup and paused Tokio time; restore intent
|
||||
contention before the first authority read, with one persisted tuple and no retry.
|
||||
The terminal checker fixture keeps its original Tokio driver alive through accepted cleanup,
|
||||
then shuts down projection producers and drains actual DB jobs.
|
||||
- `worker_projection::tests`: real SQL contention, commit/payload publication,
|
||||
ordered snapshot registration, and replacement draining a cancelled consumer's
|
||||
accepted SQL before applying a new Runtime snapshot.
|
||||
- Auth/proof and Runtime subscription tests protect availability classification,
|
||||
replay rejection, and no bearer fallback.
|
||||
- `workdir`'s `http-client` tests protect awaited fresh authorization and exact
|
||||
request claims before any HTTP send.
|
||||
|
||||
These fixtures use temporary authority data and observable synchronization, not
|
||||
live databases or a fixed sleep to guess completion. This change does not claim a
|
||||
latency benchmark or a measured production incident.
|
||||
@@ -0,0 +1,287 @@
|
||||
# T-748: async caller ownership inventory and scoped repair evidence
|
||||
|
||||
## Current status and evidence boundary
|
||||
|
||||
MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq13 requested changes for published
|
||||
`ebce7ab7b1230654b8eb5340afca8de835f24f12`. The independent review confirmed the
|
||||
revoke cancellation/publication fix, but found three further direct async callers.
|
||||
Earlier full suites exercised successful outcomes without proving every composed
|
||||
caller boundary. Repeatedly treating a helper's apparent purpose as its lane
|
||||
classification missed Runtime lookup in Ticket projection and post-await reads.
|
||||
A successful suite is not a substitute for the entry-to-publication inventory.
|
||||
|
||||
The seq13 and adjacent fixes were committed as `5873f250`; embedded protocol
|
||||
and Feature shutdown fixes as `4d981194`. The remaining confirmed caller-placement
|
||||
and Worker stop repairs are included in the follow-up below. The read-only audit
|
||||
children were released after result-observation gaps; that release was not treated
|
||||
as completion. No audit-to-implementation approval gate is imposed.
|
||||
No target integration, Ticket conclusion, live update or attachment cleanup is
|
||||
performed. T743 domain APIs are not copied or implemented here; their owner still
|
||||
needs normal composition and exact combined-source review.
|
||||
|
||||
The tables and line numbers below preserve the **pre-repair audit snapshot**.
|
||||
“Unfixed”, “pending”, and “remaining” in those historical sections describe that
|
||||
snapshot, not the current implementation. The final section supersedes their
|
||||
placement/stop status and explicitly distinguishes tested boundaries from limits.
|
||||
|
||||
Audited snapshot `server.rs` SHA-256:
|
||||
`dae05a2649f5d4f884e8118ba4ce6baa902de6e61727c20e97c0afa3661ef3a6`.
|
||||
Audited snapshot `worker_operations.rs` SHA-256:
|
||||
`e3ad2327370440fdc40fb2c76f9604b984667e3472570d76145f4d0987d343fd`.
|
||||
These file hashes identify the inspected implementation, not a provider review ref.
|
||||
|
||||
## Current confirmed ownership changes
|
||||
|
||||
| Entry / composed family | Authority and synchronous dependencies | Current owner / completion boundary | Evidence and limits |
|
||||
|---|---|---|---|
|
||||
| `ticket_implementation_cancel` → `scoped_cancel_ticket_implementation` | Ticket authority/assignment reads; Runtime cancel; current-session close; intact assignment cancellation transaction; detail | One accepted finite Host workflow owns cancellation through transaction and response. No Runtime wait on DB, no nested Host admission | Pre-authenticated generated POST route; real separate SQLite writer holds actual cancellation transaction; timer/control progress; cancelled waiter retains actual lease, then one operation/cleared assignment after drain. Saturated Host returns 503 without mutation. Existing live Coder cancellation policy passes |
|
||||
| `merge_request_selector_repair` → `scoped_repair_merge_request_selector` | Browser auth/explicit confirmation; MR store; current assignment; provider ref; intact repair/audit; public projection | Auth remains an owned DB read; confirmed post-auth mixed stage is one finite Host job | Browser auth is drained before real SQL/shared-mutex contention; post-auth stage stays Pending without blocking Tokio and retains cancelled Host lease. Imported NULL-selector/no-assignment fixture rejects without repair/thread effects. Host overload preserves confirmation priority. Successful provider repair is not claimed by this narrow fixture |
|
||||
| restore routes → `restore_runtime_worker_with_context` → `project_worker_restore_response_owned` | Existing authorized Host restore returns historical result; then whole stored Worker/links/Workdir/capability projection | Whole independent DB stage. A result with no Worker remains a pure response and needs no DB admission | Real SQL writer/shared-mutex hold; timer and independent Host control progress; bounded overload; cancellation retains actual DB lease; historical state preserved and no registry observation write. This tests post-restore policy, not a complete restore endpoint/E2E |
|
||||
| Ticket detail/show/query and alias `get_ticket`; edit/event detail and assignment preparation | `TicketAuthority` projection includes open-MR `RuntimeTicketSourceCommitResolver` → synchronous provider HTTP | Mixed stages use finite Host, not sole DB. Subsequent pure assignment mutation stages keep their existing owner | Detail/query held loopback `/v1/repository-refs/observe`; unrelated DB/timer/control progress; cancel retains actual Host budget; overload stays 503. Provider-placement fixture is explicitly unsigned, not proof of signed request authority |
|
||||
| Ticket tracking/case decision authority | Retained assignment/CAS/body/event checks plus `find_workspace_orchestrator` → live `RuntimeRegistry::worker` | Whole existing decision helper on finite Host; transaction and notification order unchanged | Held loopback live Orchestrator lookup for both tracking/case; independent SQL/timer/control; cancelled actual Host lease retained; failed live authority leaves Ticket unchanged. Existing non-assignee/browser bypass denial passes |
|
||||
| Browser Ready | Existing mutation/notifications, followed by detail (including source ref observation) | Mutation and detail use one accepted Host job and explicit `execute_ticket_rest_operation_on_host`, never a second Host admission | Fake promptly denies notification GET, holds the subsequent source-ref POST; exactly one Ready event/readiness is durable before held detail; SQL/timer/control progress and cancelled Host ownership persist. Rejected fixture setup/misidentified notification GET are excluded |
|
||||
| Worker control input/cancel/stop/restore source preflight | Synchronous live source Runtime lookup precedes `resolve_async` and operation admission | `authenticate_worker_mutation_source_owned` admits finite Host; already-owned synchronous callers retain the original helper | Actual control handler's held source GET proves DB/timer/control progress and cancelled finite Host budget; saturation returns availability, not source denial. The fake's lookup fails before an actual Worker cancellation; no product process is used |
|
||||
| Worker observation capture | Source lookup, initial SQL grant, control lock, live grant re-fence and target Runtime lookup, awaited cold protocol connection | Source on Host; initial grant on DB; actual re-fence/live lookup on Host owns the acquired control guard; cold protocol signing retains existing DB stage | Source/permission/cold-signing contracts remain covered separately. Full capture success/held-protocol composition is not claimed by the new source lookup fixture |
|
||||
|
||||
## Regressions and causal checks
|
||||
|
||||
New `server_residual_execution_tests.rs` contains eight focused tests, including
|
||||
three read/Ready and two decision-authority table cases. Exact old methods or the
|
||||
old direct stored-projection placement were restored diagnostically in turn for
|
||||
cancel, repair, restore, detail, tracking, and control. All six fail at their expected
|
||||
first-poll/shared-mutex/independent-SQL boundary. Five-second fixture watchdog
|
||||
release and its induced SQLite BUSY/panic are diagnostic bailout, not progress.
|
||||
The product source is restored in `finally` after each case.
|
||||
|
||||
Artifacts (local, not tracked product code):
|
||||
- `target/t748-residual-old-{cancel,repair,restore,detail,tracking,control}-red.log`
|
||||
- `target/t748-residual-final-restored-green.log`: **26 DB-boundary tests pass**
|
||||
- `target/t748-residual-decision-policy.log`: non-assignee/browser bypass policy passes
|
||||
- `target/t748-residual-root-check.log`: root compile closure passes
|
||||
- `target/t748-residual-provisional-full-suite.log`: original default-parallel Server
|
||||
**979 passed / 0 failed / 1 existing ignored**, binary11; **178.38 seconds**.
|
||||
Only the three deliberate executor unknown-outcome panic diagnostics appeared.
|
||||
No retained test/repro processes were observed after completion.
|
||||
- `target/t748-residual-provisional-workdir-{default,http}.log`: default153 and
|
||||
http-client156 tests pass. This does not complete the pending transitive audit.
|
||||
- fmt/diff and state-contract names **558 / 0 unapproved / 0 stale** pass.
|
||||
|
||||
All Cargo validation uses `target/t748-candidate-build`; no parallelism reduction,
|
||||
production budget increase, skipped tests or mutation retry was introduced.
|
||||
Initial test-only private enum/state/DTO mistakes failed compilation; missing
|
||||
repository seeding and Ready notification GET confusion were diagnosed and fixed.
|
||||
Those attempts are not semantic proof. The full-suite result above is provisional:
|
||||
it does **not** conclude the caller inventory or approval.
|
||||
|
||||
## Remaining inventory (not complete)
|
||||
|
||||
`target/t748-async-entry-candidates.tsv` records 661 lexical async-entry candidates
|
||||
from Server, Worker operations, auth, projection and subscription sources. Visible
|
||||
lane markers/await counts are discovery hints, not an AST/type-checked call graph
|
||||
or proof that all transitive operations are safe. Forwarders, actual handlers,
|
||||
service adapters and callback/background callers must be matched by name and
|
||||
traced through their real dependencies before marking a family inspected.
|
||||
|
||||
| Inventory work | Current status / next evidence |
|
||||
|---|---|
|
||||
| Ticket/MR/Objective/Memory/Subjektiv/Backend Job: canonical/alias/browser/Runtime entries, assignment/routing/decision/projection/publication and callback edges | First committed child result inspected. Major handler owner table returned, but follow-up final result was not observable. Parent released the child and is directly checking remaining adapters/forwarders/launch edges. Macro trait dispatch and representative Objective/Memory/Ticket forwarding plus legacy Memory/cleanup owners inspected; complete per-operation mapping remains open |
|
||||
| Auth/Runtime/Worker/control/observation/resources/Workdir/repository/Drive/settings/config/catalog/uploads/sessions/subscriptions and lifecycle background edges | First committed child result inspected; parent verified the residual structures below against actual source. The follow-up result was not observable after two Idle notifications; parent released the read-only child and directly traced the remaining four edge groups below. This is not delegated completion evidence |
|
||||
| Existing pure DB, mixed Host, mandatory Subject callback and accepted publication boundaries | Earlier documented evidence remains relevant; unchanged code does not exempt newly found composed callers from tracing |
|
||||
| `WorkspaceWorker::stop` first Host result → waiter-owned session close → second Host stage | Parent verified the split in actual source. Structural ownership gap, not yet a cancellation reproducer; tracked below rather than silently marked safe |
|
||||
|
||||
## Additional residuals found before republication (unfixed)
|
||||
|
||||
The first two read-only audit results were inspected through their committed
|
||||
sessions. The parent then read the concrete handler/helper source listed below.
|
||||
These are **static execution-placement/continuation findings**, not new successful
|
||||
endpoint, cancellation or overload tests. None has been repaired or published in
|
||||
this snapshot. The provisional suite above therefore cannot establish safety for
|
||||
these families. Lower recovery/transport edges are still being traced by the
|
||||
parent after releasing the read-only children; this is not independent review.
|
||||
|
||||
`S` below means `crates/workspace-server/src/server.rs` at the frozen hash;
|
||||
`O` means `crates/workspace-server/src/server/worker_operations.rs`.
|
||||
|
||||
| Entry / family | Entry → authorization → synchronous work → completion | Verified residual and required boundary |
|
||||
|---|---|---|
|
||||
| `repository_access_projection`, `repository_ssh_credential_{list,create,generate,get,delete,public_key,rotate}`, `repository_ssh_host_trust_{list,put,get,delete}` | ServerApi entries `S:7879–8107` directly await handlers `S:12934–13129`; owner authentication is awaited first, then active config projection / secret / trust / CAS / audit helpers execute synchronously | Auth ownership does not carry past await. Parent read all handlers and actual `repository_access.rs:391–436` connection reads. Whole existing synchronous operation needs an owner; pure DB and any external provider work must be classified by dependency, not by the SSH label |
|
||||
| `repository_create` → `scoped_create_repository` | Owner authentication await → validation → `config_store.insert_repository` → intent/replay classification → response, `S:23157–23214` | Insert is outside an accepted job. Preserve the existing intact insert/intent/commit helper and successful replay vs conflicting intent semantics |
|
||||
| `repository_ssh_connection_probe`, `repository_ssh_host_trust_confirm` | Owner and active binding awaits → catalog / `probe_repository_ssh_connection` → Runtime observation → trust read/CAS/write → response, `S:23247–23358` | Post-auth mixed work has no finite Host owner. Confirm must keep observed host-key selection and fingerprint comparison before the existing trust mutation |
|
||||
| `worker_session`, `worker_session_history`, `scoped_worker_session_attachment` | Workspace checks → synchronous Runtime Session API → response/body, `S:8696–8769,12011–12069`; registry facade `hosts.rs:2129–2195` | No finite Host admission at these request boundaries. Binding/signing and remote transport dependencies must stay off Tokio and out of the sole DB lane; preserve Session diagnostics and response limits |
|
||||
| `runtime_worker_attachment_upload` → `upload_runtime_worker_file_contract` | Grant claim → registry resolution → Runtime upload → grant completion or cancellation artifact deletion, `S:28081–28152` | Whole body-upload operation is still direct synchronous mixed work, distinct from the already-owned grant/cancel/delete routes. Admission must precede grant claim; accepted upload must own completion and cancellation cleanup |
|
||||
| Drive Worker authorization / `drive_grant_create` | `server_drive_grants.rs:54–74,126–166`: source/header checks → `current_worker_identity` or live Runtime worker → later DB grant precheck/mutation | Pre-DB source/live Runtime lookup has no Host owner. `S:18244–18292` confirms `current_worker_identity` directly reaches synchronous `runtime.worker`. Retain no-actor-fallback policy; owner unavailability must not be misreported as authority denial |
|
||||
| `current_worker_workspace_config_{get,attach,observe,read,commit}` | `S:11625–11699` directly invokes `workspace_config::identity`; `server_workspace_config.rs:65–74` reaches the same synchronous Runtime source helper before later owned DB stages | All five logical-config entries share the unowned identity preflight. Later accepted persistence/publication does not cover this earlier stage |
|
||||
| `workspace_workdir_list` → `scoped_list_working_directories` | DB job `S:25943–25959` → summaries `32974–32999` → all Runtime workdir observations `33515–33524` → list/detail Runtime calls and SQL reconciliation `33421–33467` | Mixed Runtime/SQL helper is incorrectly on the sole DB worker. Preserve live reconciliation and diagnostics, but move the whole mixed stage to finite Host rather than pretending the list is stored-only |
|
||||
| `workspace_config_tree_commit` → `scoped_commit_workspace_config_tree` | DB job `S:13444–13460` → prepare → repository-access projection → config commit → Runtime prompt publication, `13498–13517`; remote publication `hosts.rs:5090–5099` | SQL commit is followed by synchronous network publication inside sole DB job. Keep commit-before-best-effort-publication and accepted ownership; do not leave the continuation on the waiter. Logical config's separate owned Host dispatch is not this UI path |
|
||||
| `repository_{list,detail,log}` and aliases | DB jobs `S:29108–29183` → live repository reader → summary/log provider; `repositories.rs:276–375` invokes Git subprocess for materialized local repositories | Provider work is not pure DB. Mixed catalog/provider work needs finite Host, without changing existing diagnostics for unmaterialized sources |
|
||||
| `worker_control_stop`, `runtime_worker_stop`, `runtime_worker_stop_alias` → `WorkspaceWorker::stop` | `O:379–425`: grant/lock/re-fence → Host #1 Runtime stop returns result/control guard → waiter session lock/close → Host #2 linked Workdir observation/SQL | Structural ownership gap confirmed by parent: accepted Runtime stop has no owner for session close and subsequent Workdir synchronization if its waiter vanishes. `S:18609–18619` awaits real session close; the second Host admission also occurs after side effects. Whole accepted stop must own this continuation without nested Host acquisition. Cancellation reproduction remains pending |
|
||||
|
||||
### Parent takeover of the four remaining Runtime edge groups
|
||||
|
||||
The Runtime child's first result remains useful, but its follow-up final result
|
||||
was not available through authorized observation. The parent did not infer
|
||||
completion from Idle and did not loop Submit or inspect raw session storage.
|
||||
See `docs/report/2026-10-12-worker-observation-followup-gap.md` for the bounded
|
||||
observations and handling. The parent read the following actual source directly:
|
||||
|
||||
| Edge | Source and actual stages | Classification / unresolved contract |
|
||||
|---|---|---|
|
||||
| Embedded protocol connect and method pump | `S:31845–31902` directly calls synchronous `attach_worker_protocol_scoped` before spawning its async pump; the pump directly calls `send_connected_protocol_method_scoped`. `worker-runtime/src/runtime.rs:3486–3598` takes synchronous operation/state locks; Shutdown performs `begin_stop_operation` and `commit_stopped_worker` (`4359–4445`), with durable record/snapshot writes in fs-store builds (`5256–5269`) | **Additional confirmed placement residual**: Backend async connect/pump is not an accepted finite Host owner. The existing endpoint identity/settled-operation checks and commit/publication helpers must remain intact; no Runtime-store refactor is needed |
|
||||
| Workdir creation reservation → Runtime result → finalization | `S:26316–26609`: DB preparation/reservation, Host runtime selection/observation/create, then waiter-admitted failure or final registry/receipt DB jobs. Same-id reentry observes the reserved Runtime Workdir before attempting create (`26473–26517`) | **Confirmed continuation split**, with durable same-operation recovery intent, not automatic mutation retry. Existing recovery does not make the original create job own finalization after its waiter disappears. Final classification/regression must cover accepted Runtime outcome, final receipt, cancellation and overload without creating a second provider operation |
|
||||
| External grant fence → attachment/provider cleanup | `S:25595–25709`: DB authority/read or revoke/fence/audit, then waiter-owned cleanup/provider stop; terminal GET/revoke explicitly retries cleanup. Cleanup itself commits pending, awaits release, then admits completion/audit DB (`25719–25799`). Expiry task retains its async owner and retries (`24790–24854`) | **Confirmed continuation split plus explicit recovery**, not a claim that revocation failed or is unsafe after its DB fence. Grants remain denied by their committed fence. Need causal cancellation/lease evidence for cleanup and preserved retry/expiry policy before choosing the smallest ownership fix |
|
||||
| Command provider acceptance → registration/audit | `S:19507–19631`: Host identity, DB attachment fence, async session/provider, then synchronous command-handle registration. External `operate` admits audit-requested DB, sends provider request with explicit cancel guard, receives result, then admits audit-completed DB (`888–1024`); External command-handle registration follows another await (`1372–1428`). Local command start retains Runtime resources in its actual task and owns terminal-close admission (`workdir/src/local.rs:1831–1897`) | **Partially protected / continuation question**. SQL jobs are owned and local actual command resources are retained; explicit external cancel is not SQL retry. Provider acknowledgement-to-registration can cross a cancellable audit await. Do not assert an orphan process or whole command success without a focused protocol/registration reproducer; do not rewrite external cancellation semantics speculatively |
|
||||
| Catalog and single-workspace shutdown | `S:12222–12279`: catalog waits for HTTP serve, drains Worker projection and owns Feature shutdown DB (`2651–2664`); single-workspace `serve` aborts its hook but calls synchronous Feature shutdown directly. `feature-storage/src/lib.rs:220–241,521–523` fences and waits on connection close. Executor `drain` (`db_executor.rs:290–315`) covers only jobs accepted before its first poll, not later submissions | **Additional confirmed direct shutdown placement residual** in single-workspace serve. Separately, catalog does not demonstrate hook/expiry/Host/cleanup producer quiescence before storage shutdown. Whole producer/drain ordering remains a lifecycle question; no live shutdown was invoked to test it |
|
||||
|
||||
Domain generated adapters, remaining Tool/WIP forwarding, Backend Job / legacy
|
||||
Memory launch lower edges remain under follow-up inspection. Existing domain SQL
|
||||
helpers are not being reimplemented or globally refactored; ownership tracing is
|
||||
separate from re-auditing every SQL semantic branch. None of the additional rows
|
||||
above has new executable regression evidence yet.
|
||||
|
||||
|
||||
T743's additional assignment page/diagnostics/answer APIs are absent from this
|
||||
source. Ticket seq8–9 identifies their separate owner's normal composition with
|
||||
the published T748 facade. This inventory is **not** an attestation of that future
|
||||
combined source; no duplicate API or combined approval is introduced here.
|
||||
|
||||
## Republication conditions
|
||||
|
||||
Review-ready requires a consolidated caller-name list with each entry's auth,
|
||||
SQL/provider/Runtime stage, post-await projection, post-commit publication and
|
||||
owner identified; confirmed residuals repaired with focused causal regressions;
|
||||
remaining uncertainty explicit; and final root/changed-crate/affected-feature,
|
||||
format/diff/name checks on that final source. Only then normal same-selector push,
|
||||
provider exact-HEAD verification and fresh trusted independent review of this MR.
|
||||
No new Reviewer Runtime or separate Coder is created. Read-only audit children do
|
||||
not grant review verdict or integration authority.
|
||||
|
||||
## Scoped caller repair and validation history
|
||||
|
||||
The same Ticket branch is retained for advancing the existing MR. This section
|
||||
records validation history, not approval or integration. The final-validation
|
||||
section below supersedes the intervening failed runs without deleting their evidence.
|
||||
The existing domain helpers,
|
||||
authorization fences, receipt/replay rules, and transaction/publication order
|
||||
remain intact. No automatic mutation retry, production limit increase, nested
|
||||
Host admission, or test parallelism reduction was introduced.
|
||||
|
||||
| Repaired family | Current owner / executable evidence |
|
||||
|---|---|
|
||||
| Repository access/SSH credential and host-trust CRUD, Repository registration | Whole post-owner-auth synchronous operation on DB. Registration regression observes the **config store's actual connection** waiting behind a separate SQLite writer, then cancels the waiter; after drain exactly one Repository exists and the same registration intent returns HTTP200/replayed without another row |
|
||||
| SSH probe/host-trust confirm | Whole post-owner/binding-auth mixed probe/observed-key selection/fingerprint comparison/trust mutation on Host. Existing narrow SSH/trust policy tests pass in the full Server suite; no successful real SSH connection or host-key exchange is claimed |
|
||||
| Repository list/detail/log and aliases | Live Git/catalog helpers on Host, not DB. With DB admission saturated, empty catalog and absent-key detail/log retain their ordinary success/404 responses. This regression proves lane selection, not held Git subprocess behavior |
|
||||
| Worker Session/history/attachment | Typed Session service and attachment handler admit Host before synchronous Runtime observation. Loopback held Runtime lookup proves independent SQL/timer/control progress, cancelled actual lease retention, bounded overload and release after actual drain |
|
||||
| Body upload | Host admission precedes the upload grant claim and owns resolution/upload/completion/cancellation cleanup. The same loopback table holds the Runtime lookup after claim; cancelled waiter leaves actual owner, and provider refusal completes the claim back to Pending. Successful artifact upload/delete and established remote sessions are not exercised by this refusal fixture |
|
||||
| Drive Worker authorization / grant create; all five logical-config identity entries | Live source/target preflight uses Host; later DB authorization and mutations retain their previous transaction owner. Three held-lookup table cases prove independent SQL/timer/control, cancelled lease retention, and typed HTTP503 availability (not false authority denial) when Host admission is full. Existing no-actor-fallback and source-fence policies remain in the suite |
|
||||
| Workdir list | Whole live Runtime reconciliation/SQL helper on Host. Held Runtime list lookup in the Session/Workdir/upload table proves sole DB remains free and cancellation retains the actual Host budget |
|
||||
| UI config commit | Whole existing evaluate/project/commit/best-effort prompt publication helper on Host. Loopback held prompt POST confirms the changed tree is already durable while independent DB/timer/control proceeds. Cancelling the waiter retains publication ownership; refused publication neither repeats nor rolls back the commit |
|
||||
| Worker stop canonical/alias/control entries | One accepted Host workflow owns Runtime stop, session fence/close, stored read and linked Workdir synchronization. An in-process Runtime emits its stop receipt while a real session fence is held; after waiter abort and fence release the registered session is removed. Restoring the exact old stop method makes this test fail at the missing cleanup assertion; restoring the fix passes. No real Worker/product process is spawned or stopped. Linked Workdir provider success is not claimed by this session-only fixture |
|
||||
| Embedded protocol connect/method; single-workspace Feature shutdown | `4d981194` owns these operations on the existing finite Host/DB lanes. Focused protocol and actual Feature connection shutdown regressions remain green in the final suite |
|
||||
|
||||
Final logs in the Workdir (build artifacts, not committed product content):
|
||||
- `target/t748-stop-old-placement-red.log`: old stop fails **missing cleanup**, not timeout.
|
||||
- `target/t748-stop-restored-green.log`: restored accepted-stop ownership passes.
|
||||
- `target/t748-caller-host-green.log`: Session/history/attachment/Workdir/upload table passes.
|
||||
- `target/t748-repository-create-green.log`: actual SQLite commit/cancel/replay passes.
|
||||
- `target/t748-drive-config-preflight-green.log`: all three live authority cases pass.
|
||||
- `target/t748-ui-prompt-owner-green.log`: commit-before-publication passes.
|
||||
- `target/t748-repository-provider-green.log`: all three Repository read lanes pass.
|
||||
- `target/t748-caller-final-root-check.log`: workspace-root `cargo check` passes.
|
||||
- `target/t748-caller-final-server-suite.log`: original default-parallel Server
|
||||
**988 passed, 0 failed, 1 existing ignored**, plus **11 binary tests**, 185.38s.
|
||||
- `target/t748-caller-final-workdir-default.log`: **153** tests pass across targets.
|
||||
- `target/t748-caller-final-workdir-http.log`: **156** tests pass across targets.
|
||||
- fmt/diff checks pass; `target/t748-caller-final-names.log`: **558 / 0 unapproved / 0 stale**.
|
||||
|
||||
Test setup errors are excluded from semantic evidence: the first stop fixture
|
||||
omitted the Runtime's live Worker summary; the Repository busy observer initially
|
||||
watched the general store rather than the config store's distinct connection.
|
||||
Both were corrected before the successful runs. Missing package/Python command
|
||||
and compile-time DTO/Duration/Drive-error mistakes also prove no product behavior.
|
||||
|
||||
### Explicit limits, not an expanding audit gate
|
||||
|
||||
The four lower-edge rows above separate confirmed direct placement repairs from
|
||||
Workdir create / external grant cleanup / provider acknowledgement continuation
|
||||
questions and catalog producer quiescence. The latter are not redesigned here,
|
||||
per the user's instruction not to expand speculative Workdir/grant or unrelated
|
||||
Feature work indefinitely. Their existing durable recovery, explicit cleanup
|
||||
retry and provider cancellation policy are preserved, not reclassified as
|
||||
end-to-end cancellation proof. This report is not a mechanically complete
|
||||
transitive call graph or an assertion that every future/composed caller is safe.
|
||||
T743's separate new API composition still requires its owner's exact combined
|
||||
source validation/review. None of these limits constitutes Ticket conclusion,
|
||||
MR approval, target integration or permission to update the running environment.
|
||||
|
||||
### Post-suite focused validation blocker
|
||||
|
||||
After the successful full suite above, the current candidate was checked with
|
||||
`cargo test -p yoi-workspace-server --lib db_execution_tests` using the same
|
||||
candidate target directory and ordinary parallel execution. Result: **31 passed /
|
||||
1 failed**, in 8.06s. Failure:
|
||||
`cancelled_runtime_key_revoke_commits_audit_and_removes_broker_registration`
|
||||
reported the shared `observed` helper's two-second `Elapsed(())` timeout, without
|
||||
identifying which of its several lifecycle waits timed out. Source inspection
|
||||
confirms projection shutdown is already present in this fixture; it is not
|
||||
legitimate to attribute the failure to omitted projection shutdown, treat it as
|
||||
load-only, increase limits, reduce parallelism, or rerun until green.
|
||||
|
||||
Evidence is preserved in `target/t748-caller-final-boundaries.log`. The prior
|
||||
full-suite success is real but insufficient to declare final validation complete.
|
||||
The new caller and stop cases all passed in this focused run. Next diagnosis must
|
||||
identify the exact wait and inspect actual HTTP/job/broker ownership events before
|
||||
choosing a fixture or product repair. No normal push, fresh review, approval,
|
||||
integration or Ticket conclusion was performed for this candidate.
|
||||
|
||||
## Continued diagnosis and final validation
|
||||
|
||||
Returning the failed test as a stop condition was a workflow mistake, not an
|
||||
external outage, lost authority, or additional audit approval gate. Work continued
|
||||
on the same branch. The failed run above is retained as evidence.
|
||||
|
||||
The `observed` helper now reports its caller location. A diagnostic 32-case run
|
||||
passed; that result alone was **not** treated as a diagnosis or used to erase the
|
||||
failure. Source tracing identified an uncontrolled cold broker signing producer
|
||||
in the revoke fixture. The test now receives the actual signed WebSocket
|
||||
handshake and drains its completed DB work **before** holding the revoke writer.
|
||||
The handshake remains pending, so broker shutdown is still tested. An early HTTP
|
||||
response is selected against actual SQLite entry, reporting status/body rather
|
||||
than waiting for an event that can no longer occur. The original failed run did
|
||||
not identify its exact wait; attributing it conclusively to signing would exceed
|
||||
the captured evidence.
|
||||
|
||||
Subsequent full runs exposed the Ready table's distinct failure **before** source
|
||||
POST arrival (caller location, not timer/control failure). Artifacts:
|
||||
`target/t748-post-fixture-server-suite.log` and
|
||||
`target/t748-final-framing-server-suite.log`. The HTTP fake previously assumed
|
||||
one TCP read contained the request line and answered without consuming a body.
|
||||
It now parses the complete header/body; a deterministic two-byte-fragment reader
|
||||
proves complete frame consumption and truncated-body rejection. This is a real
|
||||
fixture flaw but does not by itself establish the historical timeout cause.
|
||||
|
||||
Ready setup includes mutation and best-effort notification before the source
|
||||
POST. The held-provider arrival now uses the existing five-second fixture
|
||||
watchdog as its setup bound, selected against handler completion. The subsequent
|
||||
**timer and independent SQL/control observations retain their two-second budget**;
|
||||
actual held-request cancellation/lease/overload/cleanup assertions are unchanged.
|
||||
Production admission limits, normal test parallelism, and provider timeout are
|
||||
unchanged. This setup-bound change is explicit, not a hidden limit increase or
|
||||
claim that Ready completes within two seconds under a whole-suite workload.
|
||||
Restoring the exact published old Ready method still fails this revised test
|
||||
(`target/t748-ready-old-boundary-red.log`, Ready returns only after synchronous
|
||||
provider wait); restored code passes, so the offload distinction is not weakened.
|
||||
|
||||
Final code validation after these fixture changes:
|
||||
- `target/t748-boundary-setup-fixed-green.log`: **33 DB-boundary cases pass**.
|
||||
- `target/t748-final-setup-root-check.log`: root compile closure passes.
|
||||
- `target/t748-final-setup-server-suite.log`: original default-parallel Server
|
||||
**989 passed / 0 failed / 1 existing ignored**, **11 binary tests**, 189.66s.
|
||||
- `target/t748-final-setup-workdir-default.log`: **153 pass** across targets.
|
||||
- `target/t748-final-setup-workdir-http.log`: **156 pass** across targets.
|
||||
- `cargo fmt --all -- --check`, `git diff --check HEAD`, and state-contract names
|
||||
**558 / 0 unapproved / 0 stale** pass.
|
||||
|
||||
No failed semantic run was merely repeated until green: the intervening source
|
||||
changes and the limits of each diagnosis are recorded above. These results support
|
||||
publication for fresh exact-source independent review, not reviewer approval,
|
||||
MR integration, Ticket conclusion, or updates to the live dogfooding environment.
|
||||
@@ -0,0 +1,444 @@
|
||||
# T-748: default-parallel suite failure diagnosis
|
||||
|
||||
## Sources and preserved observations
|
||||
|
||||
- Original local candidate: `61533fadb6b39465af60cb55aed2b4d0650319cd`.
|
||||
- Comparison source: `56795fb92cb9def9bcd701bd51d92e99526f8eff` (develop/T742).
|
||||
- Same implementation branch: `work/T-748-bounded-db-execution`.
|
||||
- The original `target/t748-server-tests.log` remains untouched. Its 97 FAILED
|
||||
lines and 600-second timeout were symptoms, not proof of one cause.
|
||||
- Initial diagnosis used `cargo test -p yoi-workspace-server -- --nocapture`,
|
||||
default libtest parallelism, without changing queue limits, features, or startup
|
||||
settings. `target/t748-diagnostic-environment.txt` records the environment;
|
||||
this host reports 24 available CPUs. Diagnostic captures were bounded to
|
||||
90/150 seconds to obtain errors before the original timeout, not counted as
|
||||
successful full-suite runs.
|
||||
|
||||
Logs under `target/` are local diagnostic artifacts, not tracked review inputs.
|
||||
This report records their meaning and the permanent regressions that reproduce it.
|
||||
|
||||
## 1. Different fixtures consumed one global Host admission budget
|
||||
|
||||
The first non-intentional errors in `target/t748-server-nocapture.log` were
|
||||
`DbCapacityExceeded` in manual Worker setup (`server.rs` at the then-current
|
||||
line 50614), followed by assignment fixture construction (52990/53045) and
|
||||
HTTP 503 responses. The deliberate before/after-commit executor panic tests
|
||||
are different: those passed and verified unknown-outcome diagnostics.
|
||||
|
||||
Temporary submit/start/cleanup instrumentation recorded submitting test names,
|
||||
closure types, actual job IDs and the execution-owner identity. It showed a
|
||||
single Host lane with `max=18`, with accepted jobs from unrelated fixtures.
|
||||
A rejection snapshot contained `{32,45,46,...,61}`. The complete diagnostic
|
||||
capture later contained cleanup for every submitted Host job on that owner;
|
||||
it did not show permanently abandoned accepted Host jobs. Instrumentation was
|
||||
removed before the source was committed/published.
|
||||
|
||||
`target/t748-live-process-18.txt` sampled the test process and threads at the
|
||||
actual rejection. That sample had 126 threads; the Host workers were runnable,
|
||||
and the process was using approximately 17 CPU cores. There was no captured
|
||||
thread-creation EAGAIN/unavailable error. This evidence does not establish an
|
||||
OS thread/CPU hard limit as the cause. No test processes remained after the
|
||||
bounded diagnostic run or the completed suite.
|
||||
|
||||
A representative failure passed alone (0.56 seconds); the eight logging cases
|
||||
passed with four test threads (2.46 seconds). The isolated comparison source's
|
||||
full default-parallel suite passed: 931 library tests, one pre-existing ignored
|
||||
migration fixture, 11 binary tests; library duration 168.74 seconds. This did
|
||||
not prove that all new failures were capacity-related, but did distinguish the
|
||||
new shared-admission rejection from an observed baseline failure.
|
||||
|
||||
### Ownership fix (`f3d4aba5`)
|
||||
|
||||
Production budgets remain shared and finite: DB 1 worker/32 queued, Host 2/16,
|
||||
cleanup 2/16. Ordinary fixtures now explicitly own a finite DB executor and lazy
|
||||
Host/cleanup owners at those same limits. API clones, child Workspace APIs, and
|
||||
weak retention adapters use their injected owner. No global fixture lane cache,
|
||||
parallelism reduction, unlimited queue, or capacity-wait retry was added.
|
||||
|
||||
The permanent saturated-Host regression fills both running slots and all 16
|
||||
queued slots, cancels waiters, advances paused Tokio time, and still observes
|
||||
rejection until actual completion. Another fixture and the separate cleanup lane
|
||||
can progress; draining restores the original budget and completes exactly 18
|
||||
accepted jobs. Existing real-SQLite cancellation/lease/drain tests remain active.
|
||||
|
||||
## 2. SQL offload invalidated fixture assumptions about first poll and publication
|
||||
|
||||
After Host isolation the default-parallel suite finished in 171.57 seconds:
|
||||
954 passed, three failed, one existing ignored. Actual errors were:
|
||||
|
||||
- removal subscriber saw a legitimate queued Runtime stop Upsert before Removed;
|
||||
- external session-open expiry exceeded its one-second bound while its DB lane
|
||||
was still shared across other fixtures;
|
||||
- grant replacement returned success because one initial poll had only submitted
|
||||
the offloaded authority read, not reached the held original-grant lock.
|
||||
|
||||
The ordinary DB fixtures were isolated too. The one-second expiry assertion was
|
||||
not increased. Grant testing now drains its actual authority read then polls the
|
||||
saved response to reach the held lock before replacing the grant. Removal testing
|
||||
observes the committed stopped projection before taking the removal baseline.
|
||||
The Removed payload assertion is unchanged. These are observable synchronization,
|
||||
not sleeps or retries. Focused expiry, grant, and removal tests all passed.
|
||||
|
||||
A captured cleanup job also attempted to poll a Tokio timer after its fixture
|
||||
runtime had shut down. The terminal checker fixture now retains that original
|
||||
runtime through accepted Host and cleanup jobs, then stops projection producers
|
||||
and drains actual DB work before ending. Terminal SQL state is not treated as
|
||||
actual cleanup completion. The subsequent full run contained only the three
|
||||
intentional executor panic diagnostics, not the shutdown-context panic.
|
||||
|
||||
The synchronization/teardown changes are in `df9d056d`.
|
||||
|
||||
## 3. Restore intent's DEFERRED read-to-write upgrade returned SQLite BUSY
|
||||
|
||||
The next default-parallel run finished in 171.48 seconds with 956 passed, two
|
||||
failed, one existing ignored. Both failures were SQLite `DatabaseBusy`, extended
|
||||
code 5, not capacity rejection:
|
||||
|
||||
- production-profile zero-input Orchestrator restore;
|
||||
- pending manual Worker claim restore reconciliation (500 instead of 409).
|
||||
|
||||
Both failed alone too. `target/t748-busy-origin.log` captured the error and a
|
||||
backtrace through `pin_internal_worker_restore_intent` -> the transaction helper
|
||||
-> internal restore -> manual claim. The journal read authority first in a
|
||||
DEFERRED transaction while the offloaded projection's other SQLite connection
|
||||
could write. Such an upgrade can return BUSY without invoking SQLite's busy
|
||||
handler; treating this as a retryable whole mutation would be incorrect.
|
||||
|
||||
A new real-SQLite regression holds an IMMEDIATE writer on another connection,
|
||||
registers an owned per-connection busy observer, and attempts one journal pin.
|
||||
The old helper fails deterministically with BUSY/5 before waiting for ownership
|
||||
(`target/t748-deferred-upgrade-red.log`, 0.23 seconds). The fixed helper waits
|
||||
before its first authority read, while a current-thread timer and control task
|
||||
progress. After releasing the writer it persists exactly one requested tuple;
|
||||
there is no application-level mutation retry.
|
||||
|
||||
`f597f1b5` makes this existing mutation helper explicitly
|
||||
`with_immediate_transaction`. Restore journal and deletion authority reads,
|
||||
mutation and commit remain in one transaction. Focused DB/HTTP tests (8), the
|
||||
original restore and claim cases, and deletion tests (13) passed.
|
||||
|
||||
## 4. Why a pre-dispatch failure could become the 600-second suite timeout
|
||||
|
||||
The original long-running logging case joined an HTTP request with a fake provider
|
||||
waiting for command dispatch. On pre-dispatch rejection the HTTP future can finish
|
||||
with 503 while the receiver remains Pending: its channel is still owned by the
|
||||
fixture, but no command will arrive. An unbounded `join!` waits forever in that
|
||||
state and hides the actual error behind the outer suite timeout.
|
||||
|
||||
The permanent `saturated_host_before_command_start_returns_availability_without_a_provider_operation`
|
||||
regression uses the real signed HTTP route with a controlled full Host budget. It
|
||||
observes the final 503/`db_capacity_exceeded` response and the still-live Pending
|
||||
provider receiver, with no process or command dispatch. The original logging
|
||||
fixture now asserts an early HTTP response before waiting for both branches, so
|
||||
an unexpected rejection reports its body immediately rather than hanging. This
|
||||
reproduces the causal timeout mechanism; the old log itself has no stack trace
|
||||
from the hung provider future.
|
||||
|
||||
The nine logging cases pass together at default parallelism (`49bfbfb2`).
|
||||
|
||||
## Artifact/source isolation during comparison
|
||||
|
||||
Initially the archived baseline used the existing Cargo target directory. That
|
||||
replaced a same-package artifact/fingerprint; one later post-fix representative
|
||||
command ran the baseline's 932-test binary (and `db_executor --list` had no tests).
|
||||
That result was rejected as candidate evidence. All later candidate checks/tests
|
||||
use `--target-dir target/t748-candidate-build`, separate from the baseline cache.
|
||||
This changes artifact location, not test parallelism, features or admission policy.
|
||||
The fresh candidate build listed the expected executor tests and total test count.
|
||||
|
||||
## Validation and remaining composition boundary
|
||||
|
||||
Pre-review code source `49bfbfb2` passed the full default-parallel Server suite,
|
||||
including the command-overload case: 960 passed, zero failures, one existing
|
||||
ignored migration fixture; 11 binary tests, library duration 175.29 seconds.
|
||||
The only captured panics were the three deliberate unknown-outcome tests.
|
||||
Root `cargo check` compile closure passed (including TUI/Server), as did Workdir
|
||||
default (153 tests) and `http-client` (156 tests) full suites, format/diff checks
|
||||
and state-contract names (558 matches, zero unapproved, zero stale exceptions).
|
||||
These commands used the isolated candidate artifact directory; libtest default
|
||||
parallelism and the production budgets were unchanged. The MR records the exact
|
||||
published source, which also includes this report and the boundary document.
|
||||
|
||||
## Post-publication independent review and causal fixes
|
||||
|
||||
MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq3 independently reviewed published
|
||||
source `66bb9d0b` and requested changes despite the successful original suite.
|
||||
Its two major findings exposed boundaries the earlier tests did not prove:
|
||||
|
||||
- `T748-DB-NETWORK-LANE`: mixed Subjektiv helpers could wait for synchronous
|
||||
Runtime HTTP on the sole DB worker. Fix `d3fcfb29` places session read/search,
|
||||
subject projection, memory staging, and legacy consolidation-scope helpers on
|
||||
the existing finite Host owner; pure DB-resident operations keep DB placement.
|
||||
Whole transaction helpers remain intact, and no Runtime I/O is introduced
|
||||
inside a SQL transaction.
|
||||
- `T748-ASYNC-PROTOCOL-SIGNING`: cold protocol source lookup called live binding
|
||||
checks and signing directly on Tokio. Fix `c0223a0d` awaits the owned DB stage.
|
||||
The source API now propagates failure explicitly: an invalid live binding or
|
||||
signing/admission failure cannot be hidden as `None` or a legacy bearer fallback.
|
||||
|
||||
Permanent real loopback HTTP regressions hold upstream Runtime replies during
|
||||
session read/search, subject and legacy-scope requests, while independently
|
||||
observing SQL, timer and control progress. Cancelling waiters retains the actual
|
||||
Host lifecycle lease until completion. Cold protocol source tests control another
|
||||
SQLite writer and shared connection mutex wait, and verify bounded overload,
|
||||
fresh signing and revocation without fallback. Existing queue/cancellation and
|
||||
deletion evidence now observes the actual mutation owner; the mixed fixtures
|
||||
explicitly inject a small finite Host budget. Production budgets are unchanged.
|
||||
|
||||
Focused boundary tests passed (10). Restoring the old placement produced a
|
||||
failing regression, and fixed placement passed; an earlier syntax-rejected
|
||||
instrumentation attempt is excluded from evidence. Diagnostic artifacts under
|
||||
`target/` are not production source or replacement domain implementations.
|
||||
|
||||
Exact code source `c0223a0d` then passed root `cargo check` and the original
|
||||
**default-parallel** Server suite: **962 passed, zero failed, one pre-existing
|
||||
ignored**, 11 binary tests; library duration **173.36 seconds**. Only the three
|
||||
intentional executor panic diagnostics appeared, with no residual test processes.
|
||||
Workdir default (153) and `http-client` (156) full suites, format/diff checks and
|
||||
state-contract naming (558 matching lines, zero unapproved, zero stale) passed.
|
||||
All used the isolated `target/t748-candidate-build` artifact directory. Logs:
|
||||
`target/t748-review-fix-{root-check,full-suite,workdir-default,workdir-http,format,names}.log`.
|
||||
The documentation-only publication `b10b7315` did not change that tested code.
|
||||
MR seq5 independently confirmed the two old fixes and repeated the checks above,
|
||||
but requested changes for a new mandatory-callback cycle; see the next section.
|
||||
|
||||
## Mandatory callback must be independent of its waiting Host parent
|
||||
|
||||
MR seq5 finding `T748-HOST-CALLBACK-CYCLE` traced Host create/restore -> Runtime
|
||||
bootstrap -> mandatory Session attribution -> the same finite Host lane. Two
|
||||
waiting parents could consume both workers while successfully admitted callbacks
|
||||
could not run. Cancelling a waiter correctly retains the actual parent, so it is
|
||||
not a solution. The Reviewer's compiled executor artifact proved this placement
|
||||
cycle, not a complete create endpoint/E2E reproduction.
|
||||
|
||||
Fix `a2487b57` separates store-only scope discovery into typed Subject, current
|
||||
Job consolidation and legacy consolidation authority. Session attribution accepts
|
||||
only Subject authority and executes its whole existing SQL operation on the
|
||||
independent DB lane. It rejects consolidation roles without calling Runtime.
|
||||
Mixed read/search, subject projections and allowed legacy consolidation operations
|
||||
keep their Runtime checks on Host; the previous DB/network fix is not undone.
|
||||
|
||||
The same dependency audit identified Subject resident context reads used by
|
||||
Runtime bootstrap/restore. All Subject-only memory operations use store-only
|
||||
Subject authority and DB ownership too (resident summary/context, proposal
|
||||
validation, receipt lookup, explicit staging). Other memory operations retain
|
||||
Host ownership because legacy authority can require synchronous Runtime lookup.
|
||||
No transaction is split across an await and no production budget is increased.
|
||||
|
||||
The strengthened signed route regression holds both actual production-sized Host
|
||||
workers (2/16) in `run_host_workflow` parents and retains their create admission
|
||||
leases until callbacks finish. Resident and attribution routes authenticate real
|
||||
Runtime source proofs, check current singleton ownership, respond successfully,
|
||||
and persist both distinct Subject session attributions **before** either parent
|
||||
releases capacity. It also preserves Session callback reentry during deletion
|
||||
drain. Runtime's wait is modeled at this policy boundary; no product process or
|
||||
complete create endpoint is spawned. Separate denial coverage proves legacy
|
||||
consolidation callbacks return 403 without Runtime lookup, leave no attribution,
|
||||
and do not bypass authentication (401).
|
||||
|
||||
Only Session attribution is an allowed deletion-drain callback; that endpoint
|
||||
allowlist was not expanded. An initial fixture sent resident reads during deletion
|
||||
and correctly received 409. The fixture now tests those reads before deletion,
|
||||
then Session attribution while deletion waits. This is alignment with the existing
|
||||
fence, not a relaxed product gate or success-picking retry. A diagnostic Python
|
||||
invocation was unavailable before making any source change; only the subsequent
|
||||
Deno lane-restoration runs are causal red evidence.
|
||||
|
||||
Restoring just Session's old Host placement fails at its expected callback
|
||||
watchdog; restoring just resident memory's old Host placement fails at its own
|
||||
watchdog. Fixed placement succeeds without parent bailout. Evidence:
|
||||
`target/t748-callback-old-{session,resident}-placement-red.log`,
|
||||
`target/t748-callback-restored-green.log`. Final focused signed callback tests (5),
|
||||
DB boundaries (10), and Subjektiv regressions (25) passed.
|
||||
|
||||
Exact code `a2487b575df1f348251214c53da61595860fcb40` passed root `cargo check` and
|
||||
original default-parallel Server **963 passed / 0 failed / 1 existing ignored**,
|
||||
11 binary tests; library duration **173.01 seconds**. Only the three intentional
|
||||
executor panics appeared, with no residual test processes. Workdir default153 /
|
||||
http-client156, fmt/diff and naming558/0/0 passed. The isolated candidate cache
|
||||
remains in use. Logs: `target/t748-callback-{root-check,full-suite,workdir-default,workdir-http}.log`.
|
||||
The docs-only publication `27b6c688` changed no tested code. MR seq7 independently
|
||||
confirmed the callback fix and earlier fixes, but identified another direct async
|
||||
caller; its independent verdict was `request_changes`, not approval.
|
||||
|
||||
## Worker discovery and adjacent metadata caller audit
|
||||
|
||||
MR seq7 major `T748-ASYNC-WORKER-DISCOVERY` traced async discovery directly into
|
||||
synchronous `workers_response`. That helper acquires the shared SQL connection
|
||||
for the Worker projection snapshot, Workdir registry/links and authority projection.
|
||||
The Reviewer's independently compiled public snapshot helper reproduced a
|
||||
300.246ms shared-mutex stall while its Tokio timer could not progress. This was
|
||||
helper-boundary evidence, not an authenticated endpoint/E2E run.
|
||||
|
||||
Fix `dbcea6c3` awaits the **whole existing** `workers_response` on the independent
|
||||
bounded DB owner. Source-proof requirement, Workspace/query/cursor validation,
|
||||
visibility filtering, pagination and control-grant behavior are unchanged. The
|
||||
other production callers were traced: Worker list/resource detail already use DB;
|
||||
mixed Runtime cleanup plans use Host. No network work is added to the DB owner.
|
||||
|
||||
The adjacent metadata audit found async `workspace_runtime_resources_response`
|
||||
also directly calling synchronous Runtime summaries (including remote signed HTTP
|
||||
and its awaited DB authorization). This call now uses finite Host ownership,
|
||||
then the existing independent async binding read. Runtime list/detail share this
|
||||
boundary. Whole SQL helpers, admission budgets and authentication fallback rules
|
||||
are not changed.
|
||||
|
||||
The permanent post-auth discovery handler regression uses a separate SQLite
|
||||
IMMEDIATE writer and busy observer to hold the actual shared connection mutex.
|
||||
While the accepted discovery read is queued, current-thread timer and independent
|
||||
Host control progress. A second read receives 503/`db_capacity_exceeded`; a source
|
||||
without proof receives 403 before admission. Cancellation retains the actual
|
||||
queued read's lease/budget until result cleanup; ordinary completion preserves the
|
||||
empty filtered catalog response. Signed route proof verification remains covered
|
||||
by the existing dedicated-permission test, separately from this post-auth policy
|
||||
boundary so middleware's own DB wait cannot hide the handler bug.
|
||||
|
||||
Restoring the exact old discovery call fails its first-poll assertion after the
|
||||
fixture watchdog releases the synchronous mutex wait. Restoring only the old
|
||||
Runtime-summary placement fails because Tokio cannot observe the real loopback
|
||||
request until the upstream watchdog releases it. Fixed summary placement lets
|
||||
SQL/timer/control progress during the held signed upstream request and keeps the
|
||||
actual Host lease after cancellation. Explicit upstream-release observation
|
||||
prevents watchdog failure cleanup from masquerading as progress. Logs:
|
||||
`target/t748-discovery-old-placement-red.log`,
|
||||
`target/t748-runtime-summary-old-placement-red.log`,
|
||||
`target/t748-discovery-summary-fixed-boundaries.log` (12 passed),
|
||||
`target/t748-discovery-summary-policy.log` (4), and callbacks (5).
|
||||
|
||||
Exact code `dbcea6c34ba55453ce588d68272df238cde19f47` passed root `cargo check` and
|
||||
original default-parallel Server **965 passed / 0 failed / 1 existing ignored**,
|
||||
11 binary tests; library duration **175.00 seconds**. Only three deliberate
|
||||
executor panics appeared, with no residual test processes. Workdir default153 /
|
||||
http-client156 and fmt/diff/naming558/0/0 passed. Logs use the same isolated
|
||||
candidate cache: `target/t748-discovery-summary-{root-check,full-suite,workdir-default,workdir-http}.log`.
|
||||
Docs-only publication `b63bdb44` changed no tested code. MR seq9 independently
|
||||
confirmed these fixes, then requested changes for Runtime management's residual
|
||||
synchronous authority calls; it was not an approval of the complete source.
|
||||
|
||||
## Runtime management: authority, network, and accepted publication ownership
|
||||
|
||||
Seq9 major `T748-ASYNC-RUNTIME-MANAGEMENT` identified registration's direct
|
||||
`get_validated` signing identity read and post-ping direct live-binding fence.
|
||||
The Reviewer reproduced both exact compiled helpers waiting approximately 300ms
|
||||
on the shared SQL mutex with no Tokio timer progress (helper-boundary, not endpoint
|
||||
or E2E proof). Registration's synchronous Runtime Worker scan also requires a
|
||||
network/authority boundary; ping previously used an unbounded blocking admission.
|
||||
|
||||
Fix `925bce2a` introduces whole owned DB stages for validated active signing
|
||||
identity and the post-ping current binding fence. Runtime Worker scans and ping
|
||||
use existing finite Host admission. A changed/revoked binding still produces
|
||||
conflict rather than attesting the earlier connection; owner/key/CAS validation
|
||||
and existing whole store transactions are retained.
|
||||
|
||||
The adjacent management audit also closes cancellation separation between SQL
|
||||
commit and local Runtime publication. Registration and metadata updates now own
|
||||
mutation, expectations/registry/subscription publication, and response observation
|
||||
in one accepted Host workflow, with their SQL helper awaited on the independent
|
||||
DB owner. Public Runtime resource/detail helpers admit ownership; the already-owned
|
||||
workflows call explicit `_on_host` variants directly instead of reacquiring the
|
||||
same lane. There is no Runtime/network await inside a SQL transaction or nested
|
||||
Host admission. A cancelled waiter cannot strand a committed metadata change
|
||||
without publishing the matching binding.
|
||||
|
||||
Launch options were another mixed Runtime summary/config helper placed on sole
|
||||
DB. Its signed Runtime authorization could enqueue DB work behind its own waiting
|
||||
parent. It now uses Host, preserving SQL/control progress and finite ownership.
|
||||
The audit traced remaining signing validation/binding-fence calls: UI signing
|
||||
operations already use DB, Runtime gate calls occur on Host, and removal already
|
||||
has an owned Host workflow. DNS validation stays async; pure local registration
|
||||
construction/notification stays in the accepted publication owner.
|
||||
|
||||
Permanent regressions cover both authority stages under a real separate SQLite
|
||||
IMMEDIATE writer and shared-mutex busy observer: current-thread timer/control
|
||||
progress, overload503, cancelled actual DB lease retention/cleanup, positive
|
||||
validation and revoked binding409. An actual signed loopback ping is held while
|
||||
SQL contention starts; after its reply the real handler's post-ping fence queues
|
||||
without blocking Tokio or Host control. Another signed loopback scan proves SQL/
|
||||
timer progress and retained cancelled finite Host admission. The metadata mutation
|
||||
case cancels while its own SQL is waiting, then observes one durable update and
|
||||
an equal published binding after actual Host/DB drain. Producers are stopped and
|
||||
drained while the original fixture runtime remains alive.
|
||||
|
||||
Old identity, binding, scan and launch placements each fail their expected
|
||||
contention/watchdog boundary; fixed code passes. Logs:
|
||||
`target/t748-management-old-{identity,binding,scan,launch}-placement-red.log`,
|
||||
`target/t748-management-final-restored-green.log` (16 boundaries), registration
|
||||
policy11 and connection policy13. A zero-match launch-options exploration is not
|
||||
counted as proof; its real held-summary case is in the 16 boundaries. The first
|
||||
fixture teardown named an unavailable method and failed compilation; it was
|
||||
corrected to the existing `shutdown_async`, not counted as a semantic test run.
|
||||
|
||||
Exact code `925bce2a405b6d3255887fba88867ab8d23882dc` passed root `cargo check` and
|
||||
original default-parallel Server **969 passed / 0 failed / 1 existing ignored**,
|
||||
11 binary tests; library duration **172.25 seconds**. Only three deliberate
|
||||
executor panics appeared, with no residual test processes. Workdir default153 /
|
||||
http-client156, fmt/diff/naming558/0/0 passed. Same isolated cache; evidence logs:
|
||||
`target/t748-management-{root-check,full-suite,workdir-default,workdir-http}.log`.
|
||||
Docs-only publication `6d8c49b7` changed no tested code. MR seq11 independently
|
||||
confirmed those fixes and repeated the full validation, but requested changes for
|
||||
cancelled trust-key revocation; it did not approve the complete snapshot.
|
||||
|
||||
## Trust-key revocation must own post-commit subscription termination
|
||||
|
||||
Seq11 major `T748-RUNTIME-REVOKE-CANCEL-PUBLICATION` reproduced the authenticated
|
||||
public DELETE trust-key route waiting for an actual SQLite writer. Cancelling its
|
||||
HTTP waiter did not cancel accepted SQL: revoke, binding-ID replacement and audit
|
||||
committed, but waiter-owned expectation removal and broker unregister were lost.
|
||||
The independent artifact used the normal public Server library's inner router,
|
||||
not outer Workspace dispatch or live-stream/E2E. New signed requests still fail
|
||||
the live binding fence; that does not shut down an already connected subscription.
|
||||
|
||||
Fix `1a065db0` gives the whole key-revoke outcome one accepted finite Host owner:
|
||||
intact CAS/audit SQL on the independent DB lane, local expectation removal, broker
|
||||
unregister/Shutdown, and response observation. Owner/scope/input preflight, typed
|
||||
trust conflicts and current-binding error responses remain unchanged. Response
|
||||
reads use the already-owned `_on_host` detail helper, with no nested Host admission.
|
||||
Admission/unknown-outcome errors retain the normal typed availability mapping;
|
||||
there is no transaction split, new queue budget or automatic mutation retry.
|
||||
The adjacent key-mutation caller audit confirms registration and metadata already
|
||||
own publication and this is the sole production key-revoke caller.
|
||||
|
||||
The permanent cancellation regression exercises the **real outer Workspace
|
||||
dispatch**, owner browser cookie, generated DELETE route, whole store mutation and
|
||||
broker. Projection producers stop before the write lock is acquired, so the
|
||||
fixture-local busy observer identifies this actual revoke transaction, not startup
|
||||
SQL. Timer/control progress before the watchdog releases SQLite. After waiter
|
||||
cancellation, the real admitted RequestLease keeps deletion Pending. Releasing the
|
||||
writer and draining actual Host/DB owners persists exactly one revoke audit,
|
||||
changes binding ID, removes expectations and broker registration, and delivers
|
||||
`Closed/ServerShutdown` to the downstream. The loopback endpoint holds a WebSocket
|
||||
handshake; this proves broker termination, not an established upstream socket or
|
||||
complete product E2E. Existing broker protocol tests cover socket replacement.
|
||||
|
||||
A separate full-Host route regression observes 503 before mutation and unchanged
|
||||
binding, audit and publication. Restoring only the exact old revoke method fails
|
||||
both permanent tests: the cancelled job leaves broker registration behind, and
|
||||
under overload the old handler commits revoke before its response helper returns
|
||||
503. Fixed code passes both without retry or reduced parallelism. Evidence:
|
||||
`target/t748-revoke-final-old-publication-red.log`,
|
||||
`target/t748-revoke-fixed-boundaries.log` (18), `target/t748-revoke-policy.log` (3),
|
||||
`target/t748-revoke-binding-policy.log` (16).
|
||||
An initial counter probe outside dispatch was invalid because dispatch replaces it
|
||||
with the real RequestLease; it was corrected to observe actual deletion drain.
|
||||
A test-only overload channel type-inference compile rejection was corrected with
|
||||
an explicit unit channel. Neither setup error is counted as semantic evidence.
|
||||
|
||||
Exact code `1a065db0fc5d07e2dcc1b9dac2fcfc0607b9a23c` passed root `cargo check` and
|
||||
the original default-parallel Server **971 passed / 0 failed / 1 existing ignored**,
|
||||
11 binary tests; library duration **173.05 seconds**. Only three deliberate executor
|
||||
panic diagnostics appeared, and no retained test/repro processes were observed.
|
||||
Workdir default153/http-client156, fmt/diff and naming558/0/0 passed. All Cargo
|
||||
checks/tests use the isolated candidate cache. Logs:
|
||||
`target/t748-revoke-{root-check,full-suite,workdir-default,workdir-http,fmt,names}.log`.
|
||||
Docs-only source `ebce7ab7` changed no tested code. MR seq13 independently confirmed
|
||||
the revoke ownership fix and repeated the full checks, but requested changes for
|
||||
Ticket cancellation, selector repair and post-restore projection's async callers.
|
||||
The current uncommitted follow-up and the requested comprehensive caller audit
|
||||
are tracked in [the caller inventory](2026-10-12-t748-caller-inventory.md).
|
||||
Successful suites are not evidence that uninspected caller boundaries are safe.
|
||||
|
||||
No latency benchmark, production incident measurement, target integration,
|
||||
Ticket conclusion, or running-environment update is claimed. T743 has now
|
||||
published its caller inventory/source; its owner will compose the T748 facade and
|
||||
new callers, then obtain review of that exact combined source. T748 does not copy
|
||||
those domain APIs or wait on the former unpublished branch.
|
||||
@@ -0,0 +1,34 @@
|
||||
# T748 / T743 orchestrated composition
|
||||
|
||||
## Exact inputs and ownership
|
||||
|
||||
- Integrated standalone T743 target: `05ba6a43adafc2a74fac29295b906bcb30f83f11`.
|
||||
- Independently approved T748 source: `e436b2ae58e940db416a5d6a9c2edb1368ec7600`.
|
||||
- Shared original develop base: `56795fb92cb9def9bcd701bd51d92e99526f8eff`.
|
||||
|
||||
T743's corrective revert deliberately removed the prematurely imported T748 tree while preserving published history. Ordinary merge therefore reports historical modify/delete conflicts; explicit composition against the original develop base restores the approved T748 platform while retaining standalone T743's public APIs/UI/Tools/Prompt. The resulting commit has both real input parents: no history rewrite, substituted source candidate, or change to T743's concluded requirements. The Orchestrator owns this composition and target integration; T743 is not reopened or gated on another lane.
|
||||
|
||||
## New caller execution boundaries
|
||||
|
||||
- `ticket_assignments_page` and `ticket_assignment_diagnostics`: owned clones and reference/scope lookup plus the whole existing synchronous query helper run on the shared finite DB executor.
|
||||
- `ticket_assignment_answer`: mixed Runtime source discovery, intact source/CAS/replay/audit/receipt/commit helper, and post-commit orchestrator discovery/delivery run under one accepted finite Host owner. Runtime/network waits never occupy the sole DB worker. The transaction is not divided by awaits, nested Host admission is not introduced, and request cancellation does not cancel/replay accepted work or lose commit-before-publication responsibility.
|
||||
- Browser forwarders remain delegates to those canonical methods; no second domain authority or DTO is introduced.
|
||||
|
||||
## Orchestrator validation
|
||||
|
||||
Commands used the repository's original default parallelism and locked dependencies:
|
||||
|
||||
- Root `cargo check --locked`: pass.
|
||||
- Assignment tracking library suite: 28 pass, including public answer source/replay/CAS/reassignment and page/cursor authorization semantics.
|
||||
- DB execution regressions: 36 pass (the 33 approved upstream tests plus 3 new public caller tests).
|
||||
- Complete `yoi-workspace-server --all-targets`: library 1007 pass / 0 fail / 1 pre-existing ignored; binary 11 pass. Library completed in 189.81 seconds.
|
||||
- `workdir --all-targets`: 153 pass; with `http-client`: 156 pass.
|
||||
- Root format check and staged/unstaged diff checks: pass.
|
||||
|
||||
New public-caller tests use the real generated Browser handlers on a current-thread Tokio runtime, explicit isolated finite executor budgets, and a deterministic actual owner/connection hold. After job admission, independent control and timer progress, cancellation leaves the queued slot occupied, overload is typed HTTP 503, drain waits for actual completion, and subsequent normal lookup has reusable capacity. They do not claim successful answer commit or Runtime E2E delivery; the existing domain and real SQLite regressions cover those distinct boundaries.
|
||||
|
||||
Causal red/green: replacing only the three new adapters with their standalone T743 synchronous bodies made all 3 new tests fail after the bounded 5-second watchdog released the held connection; restoring the finite adapters made all 3 pass. Both logs remain in the integration Workdir's `target/t748-integration-public-boundary{,-red}.log`. Complete suite, DB, and Workdir logs use the same `target/t748-integration-*` prefix. An initial Workdir command used the nonexistent package name `yoi-workdir` and failed selection before running tests; the correct Cargo package is `workdir`, as subsequently validated. No failed test was retried to pick a green result.
|
||||
|
||||
## Review and integration status at publication
|
||||
|
||||
This is an Orchestrator-authored candidate for exact-source independent review, not a claim that prior T748 approval covers this new composition. The stable `work/T-748-orchestrator-composition` ref publishes the merge candidate without updating develop. The existing assigned T748 Coder receives it for a normal merge into the same immutable MR source selector and owns the trusted Reviewer/fix loop. Only the Orchestrator will update develop after fresh exact-source readiness. No replacement MR, new Reviewer Runtime Worker, target integration, live deployment, Ticket conclusion, or resource cleanup is claimed here.
|
||||
@@ -0,0 +1,49 @@
|
||||
# Internal SubWorker follow-up observation gap
|
||||
|
||||
## Observed barrier
|
||||
|
||||
During T748's read-only caller inventory, the parent could read the first Runtime
|
||||
inventory assistant result by stable entry ref
|
||||
`E01a12e76-3893-70a0-bb73-28ee0c52f87b`. It then sent a follow-up to the same
|
||||
Idle direct child, `t748-runtime-caller-inventory`, to close four remaining
|
||||
execution/ownership boundaries.
|
||||
|
||||
After a turn-ended/Idle notification, `WorkerList` confirmed Idle, but bounded
|
||||
`SearchSessionEntries` for assistant entries after follow-up user ref
|
||||
`E01a12e76-d696-73d3-80d7-b8cde9935b37` returned no entries. Tool observations
|
||||
were visible. The overview showed only the initial and first follow-up user
|
||||
anchors, not a final result. A single additional Submit asking for the existing
|
||||
findings (not re-investigation) returned success. After its turn-ended/Idle
|
||||
notification, the same bounded assistant query still returned no entries, and the
|
||||
overview still ended at the first follow-up user anchor.
|
||||
|
||||
This is an observation-surface symptom, **not proof** that an assistant result
|
||||
was generated, lost, or that the underlying investigation completed. No raw
|
||||
session storage or hidden worker state was read to bypass the projection.
|
||||
|
||||
The same symptom also occurred for `t748-domain-caller-inventory`: its first
|
||||
result `E01a12e73-7ae4-7f02-9a7e-203140971635` was readable, but after the
|
||||
follow-up turn-ended/Idle notification the overview ended at the follow-up's
|
||||
starting commentary `E01a12e75-9c19-7bd0-9284-299116a05761` and intervening tool
|
||||
records. No final follow-up result was visible. The parent released that
|
||||
read-only delegation without another Submit and took over the remaining generated
|
||||
adapter/forwarder and launch lower-edge checks. Neither `WorkerStop` tool outcome
|
||||
was treated as evidence that the investigation completed.
|
||||
|
||||
## Handling
|
||||
|
||||
The parent did not mark the delegated work or Ticket complete, publish source,
|
||||
or replace independent review with the notification. It stopped only the two
|
||||
parent-owned read-only children to release delegation and took over the open
|
||||
Runtime and domain boundaries. No live product process, parent Worker, target
|
||||
branch, or attachment was stopped or cleaned.
|
||||
|
||||
## Improvement proposal
|
||||
|
||||
Expose enough committed turn metadata through the authorized observation surface
|
||||
to distinguish a finalized assistant result, a turn ending without a result, and
|
||||
an observation capture not yet including an accepted Submit. An explicit
|
||||
completion marker tied to the visible Submit entry would prevent an Idle
|
||||
notification from being confused with inspectable evidence. Preserve append-only
|
||||
session entries and the current authorization projection; do not expose raw
|
||||
prompts/reasoning or require polling/retry loops as a workaround.
|
||||
Reference in New Issue
Block a user