fix(server): own remaining request callers and stop continuation
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -383,46 +383,38 @@ impl WorkspaceWorker {
|
||||
) -> ApiResult<WorkerLifecycleResult> {
|
||||
let guard = self.authorize_operation(context, "stop").await?;
|
||||
let worker = self.clone();
|
||||
let (result, guard) = self
|
||||
.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let result = worker
|
||||
.api
|
||||
.runtime
|
||||
.stop_worker(&worker.identity, request)
|
||||
.map_err(|e| e.into_error())?;
|
||||
Ok((result, guard))
|
||||
})
|
||||
.await?;
|
||||
if result.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(result);
|
||||
}
|
||||
parse_runtime_worker_id_for_registry(&self.identity.worker_id)?;
|
||||
let session_lock = current_worker_session_lock(&self.api, &self.identity);
|
||||
let session_guard = session_lock.lock_owned().await;
|
||||
close_current_worker_session_locked(&self.api, &self.identity).await?;
|
||||
let worker = self.clone();
|
||||
self.api
|
||||
.host_executors
|
||||
.host()?
|
||||
.run(move || -> ApiResult<_> {
|
||||
let (_control, _session) = (guard, session_guard);
|
||||
if let Some(record) = worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(worker.api.workspace_id(), &worker.identity)?
|
||||
{
|
||||
sync_linked_workdir_after_worker_stop(
|
||||
&worker.api,
|
||||
&worker.identity.runtime_id,
|
||||
&record,
|
||||
)?;
|
||||
}
|
||||
Ok(result)
|
||||
})
|
||||
.await
|
||||
// Once admitted, Runtime stop, session close and linked Workdir
|
||||
// publication have one owner. Never hand an accepted stop's guards
|
||||
// back to its cancellable waiter or acquire this Host lane again.
|
||||
run_host_workflow(self.api.host_executors.host()?, async move {
|
||||
let _control = guard;
|
||||
let result = worker
|
||||
.api
|
||||
.runtime
|
||||
.stop_worker(&worker.identity, request)
|
||||
.map_err(|error| error.into_error())?;
|
||||
if result.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(result);
|
||||
}
|
||||
parse_runtime_worker_id_for_registry(&worker.identity.worker_id)?;
|
||||
let _session = current_worker_session_lock(&worker.api, &worker.identity)
|
||||
.lock_owned()
|
||||
.await;
|
||||
close_current_worker_session_locked(&worker.api, &worker.identity).await?;
|
||||
if let Some(record) = worker
|
||||
.api
|
||||
.store
|
||||
.get_worker_registry(worker.api.workspace_id(), &worker.identity)?
|
||||
{
|
||||
sync_linked_workdir_after_worker_stop(
|
||||
&worker.api,
|
||||
&worker.identity.runtime_id,
|
||||
&record,
|
||||
)?;
|
||||
}
|
||||
Ok(result)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn cancel(
|
||||
|
||||
@@ -20,10 +20,16 @@ const SUBJECTS: &str = "/api/w/db-execution/subjektiv/subjects";
|
||||
const OBSERVE_LIMIT: StdDuration = StdDuration::from_secs(2);
|
||||
const WATCHDOG: StdDuration = StdDuration::from_secs(5);
|
||||
|
||||
async fn observed<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::time::timeout(OBSERVE_LIMIT, future)
|
||||
.await
|
||||
.expect("request did not reach the expected DB/lifecycle state")
|
||||
#[track_caller]
|
||||
fn observed<T>(future: impl Future<Output = T>) -> impl Future<Output = T> {
|
||||
let caller = std::panic::Location::caller();
|
||||
async move {
|
||||
tokio::time::timeout(OBSERVE_LIMIT, future)
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!("request did not reach the expected DB/lifecycle state at {caller}")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
@@ -954,7 +960,7 @@ impl HeldRuntimeReply {
|
||||
Self::with_held_path(Some(path.to_owned()))
|
||||
}
|
||||
fn with_held_path(path: Option<String>) -> Self {
|
||||
use std::io::{Read, Write};
|
||||
use std::io::Write;
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let (entered, received) = oneshot::channel();
|
||||
@@ -967,16 +973,9 @@ impl HeldRuntimeReply {
|
||||
return;
|
||||
};
|
||||
stream.set_read_timeout(Some(WATCHDOG)).unwrap();
|
||||
let mut buf = [0; 4096];
|
||||
let Ok(n) = stream.read(&mut buf) else { return };
|
||||
if n == 0 {
|
||||
let Ok(Some(request)) = read_runtime_fixture_request(&mut stream) else {
|
||||
return;
|
||||
}
|
||||
let request = String::from_utf8_lossy(&buf[..n])
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
};
|
||||
if path.as_ref().is_some_and(|path| !request.contains(path)) {
|
||||
// Ready's best-effort notification lookup precedes its detail
|
||||
// source observation. Deny it promptly; hold only the named stage.
|
||||
@@ -1012,6 +1011,76 @@ impl Drop for HeldRuntimeReply {
|
||||
}
|
||||
}
|
||||
|
||||
// Read one complete request before selecting a held path or denying a prior
|
||||
// notification. TCP reads are not HTTP frame boundaries; replying while a POST
|
||||
// body remains unread can also reset the connection instead of delivering 503.
|
||||
fn read_runtime_fixture_request(reader: impl std::io::Read) -> std::io::Result<Option<String>> {
|
||||
use std::io::{BufRead, Read};
|
||||
let mut reader = std::io::BufReader::new(reader);
|
||||
let mut line = String::new();
|
||||
if reader.read_line(&mut line)? == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let request = line.trim_end().to_owned();
|
||||
let mut header_bytes = line.len();
|
||||
let mut body_len = 0;
|
||||
loop {
|
||||
line.clear();
|
||||
if reader.read_line(&mut line)? == 0 {
|
||||
return Err(std::io::ErrorKind::UnexpectedEof.into());
|
||||
}
|
||||
header_bytes += line.len();
|
||||
if header_bytes > 16384 {
|
||||
return Err(std::io::ErrorKind::InvalidData.into());
|
||||
}
|
||||
if line == "\r\n" {
|
||||
break;
|
||||
}
|
||||
if let Some((name, value)) = line.split_once(':') {
|
||||
if name.eq_ignore_ascii_case("content-length") {
|
||||
body_len = value
|
||||
.trim()
|
||||
.parse::<usize>()
|
||||
.map_err(|_| std::io::ErrorKind::InvalidData)?;
|
||||
if body_len > 1024 * 1024 {
|
||||
return Err(std::io::ErrorKind::InvalidData.into());
|
||||
}
|
||||
}
|
||||
if name.eq_ignore_ascii_case("transfer-encoding") {
|
||||
return Err(std::io::ErrorKind::InvalidData.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut body = vec![0; body_len];
|
||||
reader.read_exact(&mut body)?;
|
||||
Ok(Some(request))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn held_runtime_fixture_consumes_fragmented_headers_and_body_before_selecting_the_path() {
|
||||
struct Fragmented(std::io::Cursor<Vec<u8>>);
|
||||
impl std::io::Read for Fragmented {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
let length = buffer.len().min(2);
|
||||
std::io::Read::read(&mut self.0, &mut buffer[..length])
|
||||
}
|
||||
}
|
||||
let request = b"POST /v1/repository-refs/observe HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\nnotes";
|
||||
let mut reader = Fragmented(std::io::Cursor::new(request.to_vec()));
|
||||
assert_eq!(
|
||||
read_runtime_fixture_request(&mut reader)
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("POST /v1/repository-refs/observe HTTP/1.1")
|
||||
);
|
||||
assert_eq!(reader.0.position(), request.len() as u64);
|
||||
let truncated = &request[..request.len() - 1];
|
||||
assert_eq!(
|
||||
read_runtime_fixture_request(truncated).unwrap_err().kind(),
|
||||
std::io::ErrorKind::UnexpectedEof
|
||||
);
|
||||
}
|
||||
|
||||
struct CountJobLifetime(Arc<std::sync::atomic::AtomicUsize>);
|
||||
struct CountJobLease(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl DbOperationLifetime for CountJobLifetime {
|
||||
@@ -1538,6 +1607,31 @@ async fn cancelled_runtime_key_revoke_commits_audit_and_removes_broker_registrat
|
||||
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||
)
|
||||
.unwrap();
|
||||
// Broker registration starts an independent cold signing DB job. Observe
|
||||
// its actual handshake before occupying SQL for revoke: otherwise this
|
||||
// fixture races that producer against HTTP auth in its 1+1 DB budget.
|
||||
// Keep the handshake pending so downstream shutdown is still exercised.
|
||||
let (mut upstream, _) = observed(listener.accept()).await.unwrap();
|
||||
let handshake = observed(async {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.ends_with(b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let count = upstream.read(&mut chunk).await.unwrap();
|
||||
assert_ne!(count, 0, "broker closed before sending its handshake");
|
||||
bytes.extend_from_slice(&chunk[..count]);
|
||||
assert!(bytes.len() <= 8192, "unexpectedly large handshake");
|
||||
}
|
||||
String::from_utf8(bytes).unwrap()
|
||||
})
|
||||
.await;
|
||||
assert!(handshake.starts_with("GET /v1/protocol/ws "));
|
||||
assert!(
|
||||
handshake
|
||||
.to_ascii_lowercase()
|
||||
.contains("authorization: bearer ")
|
||||
);
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
let (mut busy, waiting) = BusyObservation::install_store(fixture.store.clone());
|
||||
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
|
||||
let server = fixture.server.clone();
|
||||
@@ -1546,9 +1640,17 @@ async fn cancelled_runtime_key_revoke_commits_audit_and_removes_broker_registrat
|
||||
"/api/w/db-execution/runtimes/revoke-runtime/trust-key",
|
||||
serde_json::json!({"expected_binding_id": binding.binding_id}),
|
||||
);
|
||||
let revoke =
|
||||
let mut revoke =
|
||||
tokio::spawn(async move { dispatch_workspace_request(State(server), request).await });
|
||||
observed(waiting).await.unwrap(); // Actual authenticated revoke reached BEGIN IMMEDIATE.
|
||||
tokio::select! {
|
||||
reached = observed(waiting) => reached.unwrap(), // Actual authenticated BEGIN IMMEDIATE.
|
||||
response = &mut revoke => {
|
||||
let response = response.unwrap();
|
||||
let status = response.status();
|
||||
let body = axum::body::to_bytes(response.into_body(), 65536).await.unwrap();
|
||||
panic!("revoke returned before its SQLite wait: {status}: {}", String::from_utf8_lossy(&body));
|
||||
}
|
||||
}
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
revoke.abort();
|
||||
assert!(revoke.await.unwrap_err().is_cancelled());
|
||||
|
||||
@@ -64,8 +64,9 @@ pub(super) async fn authorized_drive(
|
||||
// incomplete, or inconsistent with its transport identity.
|
||||
let headers = current_worker_contract_headers(context)
|
||||
.map_err(|_| fail(DriveApiErrorCode::Denied))?;
|
||||
let worker = current_worker_identity(api, workspace_id, &headers)
|
||||
.map_err(|_| fail(DriveApiErrorCode::Denied))?;
|
||||
let worker = authenticate_worker_mutation_source_owned(api, workspace_id, &headers)
|
||||
.await
|
||||
.map_err(pre_error)?;
|
||||
if context.worker_source.as_ref().is_some_and(|source| {
|
||||
source.runtime_id != worker.runtime_id || source.worker_id != worker.worker_id
|
||||
}) {
|
||||
@@ -139,9 +140,14 @@ pub(super) async fn create_grant(
|
||||
}
|
||||
}
|
||||
let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id);
|
||||
let operation_api = api.clone();
|
||||
let summary = api
|
||||
.runtime
|
||||
.worker(&worker)
|
||||
.host_executors
|
||||
.host()
|
||||
.map_err(pre_error)?
|
||||
.run(move || Ok::<_, Error>(operation_api.runtime.worker(&worker)))
|
||||
.await
|
||||
.map_err(pre_error)?
|
||||
.map_err(|_| fail(DriveApiErrorCode::NotFound))?;
|
||||
if summary.workspace.workspace_id.as_deref() != Some(workspace_id) {
|
||||
return Err(fail(DriveApiErrorCode::Denied));
|
||||
|
||||
@@ -561,7 +561,14 @@ async fn ticket_source_ref_reads_and_ready_publication_leave_sql_and_tokio_free(
|
||||
},
|
||||
));
|
||||
assert!(futures::poll!(&mut response).is_pending());
|
||||
let request = observed(reply.entered.take().unwrap()).await.unwrap();
|
||||
// Ready reaches the source POST only after mutation and best-effort
|
||||
// notification. Bound this multi-stage setup with the fixture watchdog,
|
||||
// not the shorter budget used below to prove timer/independent control
|
||||
// progress while the actual provider request is held.
|
||||
let request = tokio::select! {
|
||||
request = tokio::time::timeout(WATCHDOG, reply.entered.take().unwrap()) => request.expect("source observation setup did not reach the provider").unwrap(),
|
||||
result = &mut response => panic!("{operation} returned before the held source observation: {result:?}"),
|
||||
};
|
||||
assert!(
|
||||
request.contains("/v1/repository-refs/observe"),
|
||||
"{operation}: {request}"
|
||||
@@ -902,3 +909,473 @@ async fn cancelled_feature_shutdown_keeps_sql_close_owned_without_blocking_contr
|
||||
"accepted shutdown must fence later use"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn session_workdir_and_upload_callers_keep_provider_waits_off_sql_and_tokio() {
|
||||
use server_api::ServerApi;
|
||||
for operation in ["session", "history", "attachment", "workdirs", "upload"] {
|
||||
let fixture = fixture().await;
|
||||
let mut reply = HeldRuntimeReply::new();
|
||||
install_remote_fixture(
|
||||
&fixture,
|
||||
"caller-runtime",
|
||||
&format!("http://{}", reply.address),
|
||||
false,
|
||||
);
|
||||
let worker_id = Uuid::now_v7().to_string();
|
||||
fixture
|
||||
.store
|
||||
.upsert_worker_registry(®istry_record(RuntimeWorkerRef::new(
|
||||
"caller-runtime",
|
||||
&worker_id,
|
||||
)))
|
||||
.unwrap();
|
||||
let actor = RequestActor {
|
||||
user_id: "fixture-user".into(),
|
||||
account_id: format!("account-{WORKSPACE}"),
|
||||
handle: "fixture-user".into(),
|
||||
display_name: "Fixture user".into(),
|
||||
auth_method: ActorAuthMethod::BrowserSession,
|
||||
};
|
||||
fixture.api.attachment_upload_grants.lock().unwrap().insert(
|
||||
"caller-upload".into(),
|
||||
AttachmentUploadGrant {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
runtime_id: "caller-runtime".into(),
|
||||
worker_id: worker_id.clone(),
|
||||
account_id: actor.account_id.clone(),
|
||||
file_name: "notes.txt".into(),
|
||||
media_type: "text/plain".into(),
|
||||
expires_at_ms: u64::MAX,
|
||||
state: AttachmentUploadGrantState::Pending,
|
||||
},
|
||||
);
|
||||
let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let api = fixture.api.clone();
|
||||
let task = tokio::spawn(crate::db_executor::with_operation_lifetime(
|
||||
Arc::new(CountJobLifetime(leases.clone())),
|
||||
async move {
|
||||
let service = ServerApiContractService::Workspace(api.clone());
|
||||
match operation {
|
||||
"session" => {
|
||||
let _ = service
|
||||
.worker_session(WORKSPACE.into(), "caller-runtime".into(), worker_id)
|
||||
.await;
|
||||
}
|
||||
"history" => {
|
||||
let _ = service
|
||||
.worker_session_history(
|
||||
WORKSPACE.into(),
|
||||
"caller-runtime".into(),
|
||||
worker_id,
|
||||
server_api::WorkspaceWorkerSessionHistoryQuery {
|
||||
cursor: None,
|
||||
limit: Some(1),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
"attachment" => {
|
||||
let _ = scoped_worker_session_attachment(
|
||||
State(api),
|
||||
AxumPath((
|
||||
WORKSPACE.into(),
|
||||
"caller-runtime".into(),
|
||||
worker_id,
|
||||
"session".into(),
|
||||
"attachment".into(),
|
||||
)),
|
||||
Extension(actor),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
"workdirs" => {
|
||||
let _ = scoped_list_working_directories(
|
||||
State(api),
|
||||
AxumPath(ScopedWorkspacePath {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
"upload" => {
|
||||
let _ = upload_runtime_worker_file_contract(
|
||||
api,
|
||||
actor,
|
||||
ScopedAttachmentUploadPath {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
runtime_id: "caller-runtime".into(),
|
||||
worker_id,
|
||||
upload_id: "caller-upload".into(),
|
||||
},
|
||||
b"notes",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
},
|
||||
));
|
||||
let request = observed(reply.entered.take().unwrap()).await.unwrap();
|
||||
assert!(request.contains("/v1/"), "{operation}: {request}");
|
||||
assert!(
|
||||
!reply.upstream_released.load(Ordering::SeqCst),
|
||||
"{operation}: watchdog release is not executor progress"
|
||||
);
|
||||
let store = fixture.store.clone();
|
||||
observed(
|
||||
fixture
|
||||
.executor
|
||||
.run(move || store.get_workspace_memory_settings(WORKSPACE)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
observed(tokio::spawn(async { "control" })).await.unwrap(),
|
||||
"control"
|
||||
);
|
||||
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
|
||||
let mut queued = Box::pin(fixture.host_executor.run(|| Ok::<_, Error>(())));
|
||||
assert!(futures::poll!(&mut queued).is_pending());
|
||||
let error = scoped_list_working_directories(
|
||||
State(fixture.api.clone()),
|
||||
AxumPath(ScopedWorkspacePath {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
response_json(error.into_response(), StatusCode::SERVICE_UNAVAILABLE).await;
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(
|
||||
leases.load(Ordering::SeqCst),
|
||||
1,
|
||||
"{operation}: lost provider owner"
|
||||
);
|
||||
reply.finish();
|
||||
observed(queued).await.unwrap();
|
||||
observed(fixture.host_executor.drain()).await.unwrap();
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
assert_eq!(
|
||||
leases.load(Ordering::SeqCst),
|
||||
0,
|
||||
"{operation}: retained owner after completion"
|
||||
);
|
||||
if operation == "upload" {
|
||||
assert!(
|
||||
matches!(
|
||||
fixture
|
||||
.api
|
||||
.attachment_upload_grants
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get("caller-upload")
|
||||
.unwrap()
|
||||
.state,
|
||||
AttachmentUploadGrantState::Pending
|
||||
),
|
||||
"failed Runtime lookup must finish the accepted grant claim even with a cancelled waiter"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn repository_create_keeps_cancelled_sql_commit_owned_and_replays_one_intent() {
|
||||
let fixture = fixture().await;
|
||||
let (mut busy, waiting) = BusyObservation::install_store(fixture.api.config_store.clone());
|
||||
let mut lock = WriteLock::acquire(fixture.api.config.database_path.clone());
|
||||
let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let request = || {
|
||||
Fixture::request(
|
||||
"POST",
|
||||
&format!("/api/w/{WORKSPACE}/repositories"),
|
||||
serde_json::json!({ "repository_key": "owned-repository", "source": "https://example.invalid/repo.git", "default_ref": "main" }),
|
||||
)
|
||||
};
|
||||
let mut response = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
Arc::new(CountJobLifetime(leases.clone())),
|
||||
fixture.inner_router().oneshot(request()),
|
||||
));
|
||||
tokio::select! {
|
||||
result = &mut response => panic!("repository must wait for its write transaction: {:?}", result.unwrap().status()),
|
||||
reached = observed(waiting) => reached.unwrap(),
|
||||
}
|
||||
runtime_progress_while_locked(&lock).await;
|
||||
assert_eq!(
|
||||
observed(fixture.host_executor.run(|| Ok::<_, Error>("control")))
|
||||
.await
|
||||
.unwrap(),
|
||||
"control"
|
||||
);
|
||||
drop(response);
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 1);
|
||||
lock.release();
|
||||
busy.resume();
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 0);
|
||||
let rows = fixture.store.list_repositories(WORKSPACE).unwrap();
|
||||
assert_eq!(
|
||||
rows.iter()
|
||||
.filter(|record| record.repository_key == "owned-repository")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let replay = observed(fixture.inner_router().oneshot(request()))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json(replay, StatusCode::OK).await;
|
||||
assert_eq!(body["replayed"], true);
|
||||
assert_eq!(
|
||||
fixture.store.list_repositories(WORKSPACE).unwrap().len(),
|
||||
rows.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn drive_and_config_preflight_keep_live_authority_owned_and_report_overload_as_availability()
|
||||
{
|
||||
async fn preflight(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
operation: &str,
|
||||
) -> u16 {
|
||||
match operation {
|
||||
"config" => {
|
||||
workspace_config::identity(api, context, WORKSPACE)
|
||||
.await
|
||||
.err()
|
||||
.unwrap()
|
||||
.status
|
||||
}
|
||||
"drive" => api_macros::HttpError::status_code(
|
||||
&drive_grants::authorized_drive(api, context, WORKSPACE, false)
|
||||
.await
|
||||
.err()
|
||||
.unwrap(),
|
||||
),
|
||||
"grant" => api_macros::HttpError::status_code(
|
||||
&drive_grants::create_grant(
|
||||
api,
|
||||
context,
|
||||
WORKSPACE,
|
||||
server_api::DriveGrantCreateRequest {
|
||||
runtime_id: "preflight-runtime".into(),
|
||||
worker_id: "target-worker".into(),
|
||||
access: server_api::DriveAccess::ReadOnly,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
.unwrap(),
|
||||
),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
for operation in ["config", "drive", "grant"] {
|
||||
let fixture = fixture().await;
|
||||
let mut reply = HeldRuntimeReply::new();
|
||||
install_remote_fixture(
|
||||
&fixture,
|
||||
"preflight-runtime",
|
||||
&format!("http://{}", reply.address),
|
||||
false,
|
||||
);
|
||||
let context = server_api::ServerRequestContext {
|
||||
actor: if operation == "grant" {
|
||||
Some(RequestActor {
|
||||
user_id: "fixture-user".into(),
|
||||
account_id: format!("account-{WORKSPACE}"),
|
||||
handle: "fixture-user".into(),
|
||||
display_name: "Fixture user".into(),
|
||||
auth_method: ActorAuthMethod::BrowserSession,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
},
|
||||
worker_source: None,
|
||||
runtime_source: if operation == "grant" {
|
||||
None
|
||||
} else {
|
||||
Some(server_api::ServerRuntimeSource {
|
||||
authentication_binding_id: "fixture-binding".into(),
|
||||
runtime_id: "preflight-runtime".into(),
|
||||
worker_id: Some("source-worker".into()),
|
||||
})
|
||||
},
|
||||
origin: None,
|
||||
transport_headers: Vec::new(),
|
||||
};
|
||||
let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let api = fixture.api.clone();
|
||||
let request_context = context.clone();
|
||||
let task = tokio::spawn(crate::db_executor::with_operation_lifetime(
|
||||
Arc::new(CountJobLifetime(leases.clone())),
|
||||
async move { preflight(&api, &request_context, operation).await },
|
||||
));
|
||||
let request = observed(reply.entered.take().unwrap()).await.unwrap();
|
||||
assert!(request.contains("/v1/workers/"), "{operation}: {request}");
|
||||
let store = fixture.store.clone();
|
||||
observed(
|
||||
fixture
|
||||
.executor
|
||||
.run(move || store.get_workspace_memory_settings(WORKSPACE)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
|
||||
assert_eq!(
|
||||
observed(tokio::spawn(async { "control" })).await.unwrap(),
|
||||
"control"
|
||||
);
|
||||
assert!(!reply.upstream_released.load(Ordering::SeqCst));
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 1);
|
||||
fixture.host_executor.dispatch(|| {}).unwrap();
|
||||
assert_eq!(
|
||||
observed(preflight(&fixture.api, &context, operation)).await,
|
||||
503,
|
||||
"{operation}: overload must not be authority denial"
|
||||
);
|
||||
reply.finish();
|
||||
observed(fixture.host_executor.drain()).await.unwrap();
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn ui_config_commit_publishes_after_commit_without_holding_the_sql_lane() {
|
||||
let fixture = fixture().await;
|
||||
let current = fixture
|
||||
.api
|
||||
.config_store
|
||||
.load_workspace_config(WORKSPACE)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut reply = HeldRuntimeReply::new();
|
||||
install_remote_fixture(
|
||||
&fixture,
|
||||
"prompt-runtime",
|
||||
&format!("http://{}", reply.address),
|
||||
false,
|
||||
);
|
||||
let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let mut commit = Box::pin(crate::db_executor::with_operation_lifetime(
|
||||
Arc::new(CountJobLifetime(leases.clone())),
|
||||
scoped_commit_workspace_config_tree(
|
||||
State(fixture.api.clone()),
|
||||
AxumPath(ScopedWorkspacePath {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
}),
|
||||
Json(server_api::ConfigCommitRequest {
|
||||
base_digest: current.snapshot.digest.clone(),
|
||||
entrypoints: current
|
||||
.contract
|
||||
.entrypoints
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect(),
|
||||
changes: vec![server_api::ConfigTreeChange::Create {
|
||||
path: "notes/owned-publication.txt".into(),
|
||||
content_type: server_api::ConfigContentType::Text,
|
||||
content: "committed before prompt publication".into(),
|
||||
}],
|
||||
}),
|
||||
),
|
||||
));
|
||||
assert!(futures::poll!(&mut commit).is_pending());
|
||||
let request = observed(reply.entered.take().unwrap()).await.unwrap();
|
||||
assert!(
|
||||
request.starts_with("POST "),
|
||||
"prompt publication: {request}"
|
||||
);
|
||||
let config_store = fixture.api.config_store.clone();
|
||||
let state = observed(
|
||||
fixture
|
||||
.executor
|
||||
.run(move || config_store.load_workspace_config(WORKSPACE)),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_ne!(
|
||||
state.snapshot.digest, current.snapshot.digest,
|
||||
"publication must observe the committed tree"
|
||||
);
|
||||
assert!(
|
||||
state
|
||||
.snapshot
|
||||
.entries
|
||||
.keys()
|
||||
.any(|path| path.to_string() == "notes/owned-publication.txt")
|
||||
);
|
||||
observed(tokio::time::sleep(StdDuration::from_millis(1))).await;
|
||||
assert_eq!(
|
||||
observed(tokio::spawn(async { "control" })).await.unwrap(),
|
||||
"control"
|
||||
);
|
||||
assert!(!reply.upstream_released.load(Ordering::SeqCst));
|
||||
drop(commit);
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 1);
|
||||
reply.finish();
|
||||
observed(fixture.host_executor.drain()).await.unwrap();
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
assert_eq!(leases.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.api
|
||||
.config_store
|
||||
.load_workspace_config(WORKSPACE)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.snapshot
|
||||
.digest,
|
||||
state.snapshot.digest,
|
||||
"failed best-effort publication must not repeat or roll back the commit"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn repository_provider_reads_do_not_consume_the_sql_execution_queue() {
|
||||
let fixture = fixture().await;
|
||||
let (entered, waiting) = oneshot::channel();
|
||||
let (release, permit) = mpsc::channel();
|
||||
fixture
|
||||
.executor
|
||||
.dispatch(move || {
|
||||
let _ = entered.send(());
|
||||
let _ = permit.recv_timeout(WATCHDOG);
|
||||
})
|
||||
.unwrap();
|
||||
observed(waiting).await.unwrap();
|
||||
fixture.executor.dispatch(|| {}).unwrap();
|
||||
let probe = fixture.executor.run(|| Ok::<_, Error>(())).await;
|
||||
assert!(matches!(probe, Err(Error::DbCapacityExceeded)));
|
||||
// Empty catalog / absent key are deliberate boundary inputs: provider
|
||||
// reads keep their ordinary response even when SQL job admission is full.
|
||||
let list = observed(list_repositories(State(fixture.api.clone())))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(list.0.items.is_empty());
|
||||
let detail = observed(repository_detail(
|
||||
State(fixture.api.clone()),
|
||||
AxumPath("missing".into()),
|
||||
))
|
||||
.await
|
||||
.unwrap_err();
|
||||
response_json(detail.into_response(), StatusCode::NOT_FOUND).await;
|
||||
let log = observed(repository_log(
|
||||
State(fixture.api.clone()),
|
||||
AxumPath("missing".into()),
|
||||
Query(LogQuery { limit: Some(1) }),
|
||||
))
|
||||
.await
|
||||
.unwrap_err();
|
||||
response_json(log.into_response(), StatusCode::NOT_FOUND).await;
|
||||
let _ = release.send(());
|
||||
observed(fixture.executor.drain()).await.unwrap();
|
||||
observed(fixture.host_executor.drain()).await.unwrap();
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ fn unknown() -> Failure {
|
||||
..fail(500, "outcome_unknown")
|
||||
}
|
||||
}
|
||||
pub(super) fn identity(
|
||||
pub(super) async fn identity(
|
||||
api: &WorkspaceApi,
|
||||
context: &server_api::ServerRequestContext,
|
||||
workspace_id: &str,
|
||||
@@ -70,7 +70,9 @@ pub(super) fn identity(
|
||||
validate_workspace_scope(api, workspace_id).map_err(pre_error)?;
|
||||
let headers =
|
||||
current_worker_contract_headers(context).map_err(|_| fail(403, "access_denied"))?;
|
||||
current_worker_identity(api, workspace_id, &headers).map_err(pre_error)
|
||||
authenticate_worker_mutation_source_owned(api, workspace_id, &headers)
|
||||
.await
|
||||
.map_err(pre_error)
|
||||
}
|
||||
async fn lock(
|
||||
api: &WorkspaceApi,
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
# T-748: async caller ownership inventory and scoped repair evidence
|
||||
|
||||
## Current status and evidence boundary
|
||||
|
||||
MR `01a12d3f-e90c-7b02-9ed9-cf3bd2e822a1` seq13 requested changes for published
|
||||
`ebce7ab7b1230654b8eb5340afca8de835f24f12`. The independent review confirmed the
|
||||
revoke cancellation/publication fix, but found three further direct async callers.
|
||||
Earlier full suites exercised successful outcomes without proving every composed
|
||||
caller boundary. Repeatedly treating a helper's apparent purpose as its lane
|
||||
classification missed Runtime lookup in Ticket projection and post-await reads.
|
||||
A successful suite is not a substitute for the entry-to-publication inventory.
|
||||
|
||||
The seq13 and adjacent fixes were committed as `5873f250`; embedded protocol
|
||||
and Feature shutdown fixes as `4d981194`. The remaining confirmed caller-placement
|
||||
and Worker stop repairs are included in the follow-up below. The read-only audit
|
||||
children were released after result-observation gaps; that release was not treated
|
||||
as completion. No audit-to-implementation approval gate is imposed.
|
||||
No target integration, Ticket conclusion, live update or attachment cleanup is
|
||||
performed. T743 domain APIs are not copied or implemented here; their owner still
|
||||
needs normal composition and exact combined-source review.
|
||||
|
||||
The tables and line numbers below preserve the **pre-repair audit snapshot**.
|
||||
“Unfixed”, “pending”, and “remaining” in those historical sections describe that
|
||||
snapshot, not the current implementation. The final section supersedes their
|
||||
placement/stop status and explicitly distinguishes tested boundaries from limits.
|
||||
|
||||
Audited snapshot `server.rs` SHA-256:
|
||||
`dae05a2649f5d4f884e8118ba4ce6baa902de6e61727c20e97c0afa3661ef3a6`.
|
||||
Audited snapshot `worker_operations.rs` SHA-256:
|
||||
`e3ad2327370440fdc40fb2c76f9604b984667e3472570d76145f4d0987d343fd`.
|
||||
These file hashes identify the inspected implementation, not a provider review ref.
|
||||
|
||||
## Current confirmed ownership changes
|
||||
|
||||
| Entry / composed family | Authority and synchronous dependencies | Current owner / completion boundary | Evidence and limits |
|
||||
|---|---|---|---|
|
||||
| `ticket_implementation_cancel` → `scoped_cancel_ticket_implementation` | Ticket authority/assignment reads; Runtime cancel; current-session close; intact assignment cancellation transaction; detail | One accepted finite Host workflow owns cancellation through transaction and response. No Runtime wait on DB, no nested Host admission | Pre-authenticated generated POST route; real separate SQLite writer holds actual cancellation transaction; timer/control progress; cancelled waiter retains actual lease, then one operation/cleared assignment after drain. Saturated Host returns 503 without mutation. Existing live Coder cancellation policy passes |
|
||||
| `merge_request_selector_repair` → `scoped_repair_merge_request_selector` | Browser auth/explicit confirmation; MR store; current assignment; provider ref; intact repair/audit; public projection | Auth remains an owned DB read; confirmed post-auth mixed stage is one finite Host job | Browser auth is drained before real SQL/shared-mutex contention; post-auth stage stays Pending without blocking Tokio and retains cancelled Host lease. Imported NULL-selector/no-assignment fixture rejects without repair/thread effects. Host overload preserves confirmation priority. Successful provider repair is not claimed by this narrow fixture |
|
||||
| restore routes → `restore_runtime_worker_with_context` → `project_worker_restore_response_owned` | Existing authorized Host restore returns historical result; then whole stored Worker/links/Workdir/capability projection | Whole independent DB stage. A result with no Worker remains a pure response and needs no DB admission | Real SQL writer/shared-mutex hold; timer and independent Host control progress; bounded overload; cancellation retains actual DB lease; historical state preserved and no registry observation write. This tests post-restore policy, not a complete restore endpoint/E2E |
|
||||
| Ticket detail/show/query and alias `get_ticket`; edit/event detail and assignment preparation | `TicketAuthority` projection includes open-MR `RuntimeTicketSourceCommitResolver` → synchronous provider HTTP | Mixed stages use finite Host, not sole DB. Subsequent pure assignment mutation stages keep their existing owner | Detail/query held loopback `/v1/repository-refs/observe`; unrelated DB/timer/control progress; cancel retains actual Host budget; overload stays 503. Provider-placement fixture is explicitly unsigned, not proof of signed request authority |
|
||||
| Ticket tracking/case decision authority | Retained assignment/CAS/body/event checks plus `find_workspace_orchestrator` → live `RuntimeRegistry::worker` | Whole existing decision helper on finite Host; transaction and notification order unchanged | Held loopback live Orchestrator lookup for both tracking/case; independent SQL/timer/control; cancelled actual Host lease retained; failed live authority leaves Ticket unchanged. Existing non-assignee/browser bypass denial passes |
|
||||
| Browser Ready | Existing mutation/notifications, followed by detail (including source ref observation) | Mutation and detail use one accepted Host job and explicit `execute_ticket_rest_operation_on_host`, never a second Host admission | Fake promptly denies notification GET, holds the subsequent source-ref POST; exactly one Ready event/readiness is durable before held detail; SQL/timer/control progress and cancelled Host ownership persist. Rejected fixture setup/misidentified notification GET are excluded |
|
||||
| Worker control input/cancel/stop/restore source preflight | Synchronous live source Runtime lookup precedes `resolve_async` and operation admission | `authenticate_worker_mutation_source_owned` admits finite Host; already-owned synchronous callers retain the original helper | Actual control handler's held source GET proves DB/timer/control progress and cancelled finite Host budget; saturation returns availability, not source denial. The fake's lookup fails before an actual Worker cancellation; no product process is used |
|
||||
| Worker observation capture | Source lookup, initial SQL grant, control lock, live grant re-fence and target Runtime lookup, awaited cold protocol connection | Source on Host; initial grant on DB; actual re-fence/live lookup on Host owns the acquired control guard; cold protocol signing retains existing DB stage | Source/permission/cold-signing contracts remain covered separately. Full capture success/held-protocol composition is not claimed by the new source lookup fixture |
|
||||
|
||||
## Regressions and causal checks
|
||||
|
||||
New `server_residual_execution_tests.rs` contains eight focused tests, including
|
||||
three read/Ready and two decision-authority table cases. Exact old methods or the
|
||||
old direct stored-projection placement were restored diagnostically in turn for
|
||||
cancel, repair, restore, detail, tracking, and control. All six fail at their expected
|
||||
first-poll/shared-mutex/independent-SQL boundary. Five-second fixture watchdog
|
||||
release and its induced SQLite BUSY/panic are diagnostic bailout, not progress.
|
||||
The product source is restored in `finally` after each case.
|
||||
|
||||
Artifacts (local, not tracked product code):
|
||||
- `target/t748-residual-old-{cancel,repair,restore,detail,tracking,control}-red.log`
|
||||
- `target/t748-residual-final-restored-green.log`: **26 DB-boundary tests pass**
|
||||
- `target/t748-residual-decision-policy.log`: non-assignee/browser bypass policy passes
|
||||
- `target/t748-residual-root-check.log`: root compile closure passes
|
||||
- `target/t748-residual-provisional-full-suite.log`: original default-parallel Server
|
||||
**979 passed / 0 failed / 1 existing ignored**, binary11; **178.38 seconds**.
|
||||
Only the three deliberate executor unknown-outcome panic diagnostics appeared.
|
||||
No retained test/repro processes were observed after completion.
|
||||
- `target/t748-residual-provisional-workdir-{default,http}.log`: default153 and
|
||||
http-client156 tests pass. This does not complete the pending transitive audit.
|
||||
- fmt/diff and state-contract names **558 / 0 unapproved / 0 stale** pass.
|
||||
|
||||
All Cargo validation uses `target/t748-candidate-build`; no parallelism reduction,
|
||||
production budget increase, skipped tests or mutation retry was introduced.
|
||||
Initial test-only private enum/state/DTO mistakes failed compilation; missing
|
||||
repository seeding and Ready notification GET confusion were diagnosed and fixed.
|
||||
Those attempts are not semantic proof. The full-suite result above is provisional:
|
||||
it does **not** conclude the caller inventory or approval.
|
||||
|
||||
## Remaining inventory (not complete)
|
||||
|
||||
`target/t748-async-entry-candidates.tsv` records 661 lexical async-entry candidates
|
||||
from Server, Worker operations, auth, projection and subscription sources. Visible
|
||||
lane markers/await counts are discovery hints, not an AST/type-checked call graph
|
||||
or proof that all transitive operations are safe. Forwarders, actual handlers,
|
||||
service adapters and callback/background callers must be matched by name and
|
||||
traced through their real dependencies before marking a family inspected.
|
||||
|
||||
| Inventory work | Current status / next evidence |
|
||||
|---|---|
|
||||
| Ticket/MR/Objective/Memory/Subjektiv/Backend Job: canonical/alias/browser/Runtime entries, assignment/routing/decision/projection/publication and callback edges | First committed child result inspected. Major handler owner table returned, but follow-up final result was not observable. Parent released the child and is directly checking remaining adapters/forwarders/launch edges. Macro trait dispatch and representative Objective/Memory/Ticket forwarding plus legacy Memory/cleanup owners inspected; complete per-operation mapping remains open |
|
||||
| Auth/Runtime/Worker/control/observation/resources/Workdir/repository/Drive/settings/config/catalog/uploads/sessions/subscriptions and lifecycle background edges | First committed child result inspected; parent verified the residual structures below against actual source. The follow-up result was not observable after two Idle notifications; parent released the read-only child and directly traced the remaining four edge groups below. This is not delegated completion evidence |
|
||||
| Existing pure DB, mixed Host, mandatory Subject callback and accepted publication boundaries | Earlier documented evidence remains relevant; unchanged code does not exempt newly found composed callers from tracing |
|
||||
| `WorkspaceWorker::stop` first Host result → waiter-owned session close → second Host stage | Parent verified the split in actual source. Structural ownership gap, not yet a cancellation reproducer; tracked below rather than silently marked safe |
|
||||
|
||||
## Additional residuals found before republication (unfixed)
|
||||
|
||||
The first two read-only audit results were inspected through their committed
|
||||
sessions. The parent then read the concrete handler/helper source listed below.
|
||||
These are **static execution-placement/continuation findings**, not new successful
|
||||
endpoint, cancellation or overload tests. None has been repaired or published in
|
||||
this snapshot. The provisional suite above therefore cannot establish safety for
|
||||
these families. Lower recovery/transport edges are still being traced by the
|
||||
parent after releasing the read-only children; this is not independent review.
|
||||
|
||||
`S` below means `crates/workspace-server/src/server.rs` at the frozen hash;
|
||||
`O` means `crates/workspace-server/src/server/worker_operations.rs`.
|
||||
|
||||
| Entry / family | Entry → authorization → synchronous work → completion | Verified residual and required boundary |
|
||||
|---|---|---|
|
||||
| `repository_access_projection`, `repository_ssh_credential_{list,create,generate,get,delete,public_key,rotate}`, `repository_ssh_host_trust_{list,put,get,delete}` | ServerApi entries `S:7879–8107` directly await handlers `S:12934–13129`; owner authentication is awaited first, then active config projection / secret / trust / CAS / audit helpers execute synchronously | Auth ownership does not carry past await. Parent read all handlers and actual `repository_access.rs:391–436` connection reads. Whole existing synchronous operation needs an owner; pure DB and any external provider work must be classified by dependency, not by the SSH label |
|
||||
| `repository_create` → `scoped_create_repository` | Owner authentication await → validation → `config_store.insert_repository` → intent/replay classification → response, `S:23157–23214` | Insert is outside an accepted job. Preserve the existing intact insert/intent/commit helper and successful replay vs conflicting intent semantics |
|
||||
| `repository_ssh_connection_probe`, `repository_ssh_host_trust_confirm` | Owner and active binding awaits → catalog / `probe_repository_ssh_connection` → Runtime observation → trust read/CAS/write → response, `S:23247–23358` | Post-auth mixed work has no finite Host owner. Confirm must keep observed host-key selection and fingerprint comparison before the existing trust mutation |
|
||||
| `worker_session`, `worker_session_history`, `scoped_worker_session_attachment` | Workspace checks → synchronous Runtime Session API → response/body, `S:8696–8769,12011–12069`; registry facade `hosts.rs:2129–2195` | No finite Host admission at these request boundaries. Binding/signing and remote transport dependencies must stay off Tokio and out of the sole DB lane; preserve Session diagnostics and response limits |
|
||||
| `runtime_worker_attachment_upload` → `upload_runtime_worker_file_contract` | Grant claim → registry resolution → Runtime upload → grant completion or cancellation artifact deletion, `S:28081–28152` | Whole body-upload operation is still direct synchronous mixed work, distinct from the already-owned grant/cancel/delete routes. Admission must precede grant claim; accepted upload must own completion and cancellation cleanup |
|
||||
| Drive Worker authorization / `drive_grant_create` | `server_drive_grants.rs:54–74,126–166`: source/header checks → `current_worker_identity` or live Runtime worker → later DB grant precheck/mutation | Pre-DB source/live Runtime lookup has no Host owner. `S:18244–18292` confirms `current_worker_identity` directly reaches synchronous `runtime.worker`. Retain no-actor-fallback policy; owner unavailability must not be misreported as authority denial |
|
||||
| `current_worker_workspace_config_{get,attach,observe,read,commit}` | `S:11625–11699` directly invokes `workspace_config::identity`; `server_workspace_config.rs:65–74` reaches the same synchronous Runtime source helper before later owned DB stages | All five logical-config entries share the unowned identity preflight. Later accepted persistence/publication does not cover this earlier stage |
|
||||
| `workspace_workdir_list` → `scoped_list_working_directories` | DB job `S:25943–25959` → summaries `32974–32999` → all Runtime workdir observations `33515–33524` → list/detail Runtime calls and SQL reconciliation `33421–33467` | Mixed Runtime/SQL helper is incorrectly on the sole DB worker. Preserve live reconciliation and diagnostics, but move the whole mixed stage to finite Host rather than pretending the list is stored-only |
|
||||
| `workspace_config_tree_commit` → `scoped_commit_workspace_config_tree` | DB job `S:13444–13460` → prepare → repository-access projection → config commit → Runtime prompt publication, `13498–13517`; remote publication `hosts.rs:5090–5099` | SQL commit is followed by synchronous network publication inside sole DB job. Keep commit-before-best-effort-publication and accepted ownership; do not leave the continuation on the waiter. Logical config's separate owned Host dispatch is not this UI path |
|
||||
| `repository_{list,detail,log}` and aliases | DB jobs `S:29108–29183` → live repository reader → summary/log provider; `repositories.rs:276–375` invokes Git subprocess for materialized local repositories | Provider work is not pure DB. Mixed catalog/provider work needs finite Host, without changing existing diagnostics for unmaterialized sources |
|
||||
| `worker_control_stop`, `runtime_worker_stop`, `runtime_worker_stop_alias` → `WorkspaceWorker::stop` | `O:379–425`: grant/lock/re-fence → Host #1 Runtime stop returns result/control guard → waiter session lock/close → Host #2 linked Workdir observation/SQL | Structural ownership gap confirmed by parent: accepted Runtime stop has no owner for session close and subsequent Workdir synchronization if its waiter vanishes. `S:18609–18619` awaits real session close; the second Host admission also occurs after side effects. Whole accepted stop must own this continuation without nested Host acquisition. Cancellation reproduction remains pending |
|
||||
|
||||
### Parent takeover of the four remaining Runtime edge groups
|
||||
|
||||
The Runtime child's first result remains useful, but its follow-up final result
|
||||
was not available through authorized observation. The parent did not infer
|
||||
completion from Idle and did not loop Submit or inspect raw session storage.
|
||||
See `docs/report/2026-10-12-worker-observation-followup-gap.md` for the bounded
|
||||
observations and handling. The parent read the following actual source directly:
|
||||
|
||||
| Edge | Source and actual stages | Classification / unresolved contract |
|
||||
|---|---|---|
|
||||
| Embedded protocol connect and method pump | `S:31845–31902` directly calls synchronous `attach_worker_protocol_scoped` before spawning its async pump; the pump directly calls `send_connected_protocol_method_scoped`. `worker-runtime/src/runtime.rs:3486–3598` takes synchronous operation/state locks; Shutdown performs `begin_stop_operation` and `commit_stopped_worker` (`4359–4445`), with durable record/snapshot writes in fs-store builds (`5256–5269`) | **Additional confirmed placement residual**: Backend async connect/pump is not an accepted finite Host owner. The existing endpoint identity/settled-operation checks and commit/publication helpers must remain intact; no Runtime-store refactor is needed |
|
||||
| Workdir creation reservation → Runtime result → finalization | `S:26316–26609`: DB preparation/reservation, Host runtime selection/observation/create, then waiter-admitted failure or final registry/receipt DB jobs. Same-id reentry observes the reserved Runtime Workdir before attempting create (`26473–26517`) | **Confirmed continuation split**, with durable same-operation recovery intent, not automatic mutation retry. Existing recovery does not make the original create job own finalization after its waiter disappears. Final classification/regression must cover accepted Runtime outcome, final receipt, cancellation and overload without creating a second provider operation |
|
||||
| External grant fence → attachment/provider cleanup | `S:25595–25709`: DB authority/read or revoke/fence/audit, then waiter-owned cleanup/provider stop; terminal GET/revoke explicitly retries cleanup. Cleanup itself commits pending, awaits release, then admits completion/audit DB (`25719–25799`). Expiry task retains its async owner and retries (`24790–24854`) | **Confirmed continuation split plus explicit recovery**, not a claim that revocation failed or is unsafe after its DB fence. Grants remain denied by their committed fence. Need causal cancellation/lease evidence for cleanup and preserved retry/expiry policy before choosing the smallest ownership fix |
|
||||
| Command provider acceptance → registration/audit | `S:19507–19631`: Host identity, DB attachment fence, async session/provider, then synchronous command-handle registration. External `operate` admits audit-requested DB, sends provider request with explicit cancel guard, receives result, then admits audit-completed DB (`888–1024`); External command-handle registration follows another await (`1372–1428`). Local command start retains Runtime resources in its actual task and owns terminal-close admission (`workdir/src/local.rs:1831–1897`) | **Partially protected / continuation question**. SQL jobs are owned and local actual command resources are retained; explicit external cancel is not SQL retry. Provider acknowledgement-to-registration can cross a cancellable audit await. Do not assert an orphan process or whole command success without a focused protocol/registration reproducer; do not rewrite external cancellation semantics speculatively |
|
||||
| Catalog and single-workspace shutdown | `S:12222–12279`: catalog waits for HTTP serve, drains Worker projection and owns Feature shutdown DB (`2651–2664`); single-workspace `serve` aborts its hook but calls synchronous Feature shutdown directly. `feature-storage/src/lib.rs:220–241,521–523` fences and waits on connection close. Executor `drain` (`db_executor.rs:290–315`) covers only jobs accepted before its first poll, not later submissions | **Additional confirmed direct shutdown placement residual** in single-workspace serve. Separately, catalog does not demonstrate hook/expiry/Host/cleanup producer quiescence before storage shutdown. Whole producer/drain ordering remains a lifecycle question; no live shutdown was invoked to test it |
|
||||
|
||||
Domain generated adapters, remaining Tool/WIP forwarding, Backend Job / legacy
|
||||
Memory launch lower edges remain under follow-up inspection. Existing domain SQL
|
||||
helpers are not being reimplemented or globally refactored; ownership tracing is
|
||||
separate from re-auditing every SQL semantic branch. None of the additional rows
|
||||
above has new executable regression evidence yet.
|
||||
|
||||
|
||||
T743's additional assignment page/diagnostics/answer APIs are absent from this
|
||||
source. Ticket seq8–9 identifies their separate owner's normal composition with
|
||||
the published T748 facade. This inventory is **not** an attestation of that future
|
||||
combined source; no duplicate API or combined approval is introduced here.
|
||||
|
||||
## Republication conditions
|
||||
|
||||
Review-ready requires a consolidated caller-name list with each entry's auth,
|
||||
SQL/provider/Runtime stage, post-await projection, post-commit publication and
|
||||
owner identified; confirmed residuals repaired with focused causal regressions;
|
||||
remaining uncertainty explicit; and final root/changed-crate/affected-feature,
|
||||
format/diff/name checks on that final source. Only then normal same-selector push,
|
||||
provider exact-HEAD verification and fresh trusted independent review of this MR.
|
||||
No new Reviewer Runtime or separate Coder is created. Read-only audit children do
|
||||
not grant review verdict or integration authority.
|
||||
|
||||
## Scoped caller repair and validation history
|
||||
|
||||
The same Ticket branch is retained for advancing the existing MR. This section
|
||||
records validation history, not approval or integration. The final-validation
|
||||
section below supersedes the intervening failed runs without deleting their evidence.
|
||||
The existing domain helpers,
|
||||
authorization fences, receipt/replay rules, and transaction/publication order
|
||||
remain intact. No automatic mutation retry, production limit increase, nested
|
||||
Host admission, or test parallelism reduction was introduced.
|
||||
|
||||
| Repaired family | Current owner / executable evidence |
|
||||
|---|---|
|
||||
| Repository access/SSH credential and host-trust CRUD, Repository registration | Whole post-owner-auth synchronous operation on DB. Registration regression observes the **config store's actual connection** waiting behind a separate SQLite writer, then cancels the waiter; after drain exactly one Repository exists and the same registration intent returns HTTP200/replayed without another row |
|
||||
| SSH probe/host-trust confirm | Whole post-owner/binding-auth mixed probe/observed-key selection/fingerprint comparison/trust mutation on Host. Existing narrow SSH/trust policy tests pass in the full Server suite; no successful real SSH connection or host-key exchange is claimed |
|
||||
| Repository list/detail/log and aliases | Live Git/catalog helpers on Host, not DB. With DB admission saturated, empty catalog and absent-key detail/log retain their ordinary success/404 responses. This regression proves lane selection, not held Git subprocess behavior |
|
||||
| Worker Session/history/attachment | Typed Session service and attachment handler admit Host before synchronous Runtime observation. Loopback held Runtime lookup proves independent SQL/timer/control progress, cancelled actual lease retention, bounded overload and release after actual drain |
|
||||
| Body upload | Host admission precedes the upload grant claim and owns resolution/upload/completion/cancellation cleanup. The same loopback table holds the Runtime lookup after claim; cancelled waiter leaves actual owner, and provider refusal completes the claim back to Pending. Successful artifact upload/delete and established remote sessions are not exercised by this refusal fixture |
|
||||
| Drive Worker authorization / grant create; all five logical-config identity entries | Live source/target preflight uses Host; later DB authorization and mutations retain their previous transaction owner. Three held-lookup table cases prove independent SQL/timer/control, cancelled lease retention, and typed HTTP503 availability (not false authority denial) when Host admission is full. Existing no-actor-fallback and source-fence policies remain in the suite |
|
||||
| Workdir list | Whole live Runtime reconciliation/SQL helper on Host. Held Runtime list lookup in the Session/Workdir/upload table proves sole DB remains free and cancellation retains the actual Host budget |
|
||||
| UI config commit | Whole existing evaluate/project/commit/best-effort prompt publication helper on Host. Loopback held prompt POST confirms the changed tree is already durable while independent DB/timer/control proceeds. Cancelling the waiter retains publication ownership; refused publication neither repeats nor rolls back the commit |
|
||||
| Worker stop canonical/alias/control entries | One accepted Host workflow owns Runtime stop, session fence/close, stored read and linked Workdir synchronization. An in-process Runtime emits its stop receipt while a real session fence is held; after waiter abort and fence release the registered session is removed. Restoring the exact old stop method makes this test fail at the missing cleanup assertion; restoring the fix passes. No real Worker/product process is spawned or stopped. Linked Workdir provider success is not claimed by this session-only fixture |
|
||||
| Embedded protocol connect/method; single-workspace Feature shutdown | `4d981194` owns these operations on the existing finite Host/DB lanes. Focused protocol and actual Feature connection shutdown regressions remain green in the final suite |
|
||||
|
||||
Final logs in the Workdir (build artifacts, not committed product content):
|
||||
- `target/t748-stop-old-placement-red.log`: old stop fails **missing cleanup**, not timeout.
|
||||
- `target/t748-stop-restored-green.log`: restored accepted-stop ownership passes.
|
||||
- `target/t748-caller-host-green.log`: Session/history/attachment/Workdir/upload table passes.
|
||||
- `target/t748-repository-create-green.log`: actual SQLite commit/cancel/replay passes.
|
||||
- `target/t748-drive-config-preflight-green.log`: all three live authority cases pass.
|
||||
- `target/t748-ui-prompt-owner-green.log`: commit-before-publication passes.
|
||||
- `target/t748-repository-provider-green.log`: all three Repository read lanes pass.
|
||||
- `target/t748-caller-final-root-check.log`: workspace-root `cargo check` passes.
|
||||
- `target/t748-caller-final-server-suite.log`: original default-parallel Server
|
||||
**988 passed, 0 failed, 1 existing ignored**, plus **11 binary tests**, 185.38s.
|
||||
- `target/t748-caller-final-workdir-default.log`: **153** tests pass across targets.
|
||||
- `target/t748-caller-final-workdir-http.log`: **156** tests pass across targets.
|
||||
- fmt/diff checks pass; `target/t748-caller-final-names.log`: **558 / 0 unapproved / 0 stale**.
|
||||
|
||||
Test setup errors are excluded from semantic evidence: the first stop fixture
|
||||
omitted the Runtime's live Worker summary; the Repository busy observer initially
|
||||
watched the general store rather than the config store's distinct connection.
|
||||
Both were corrected before the successful runs. Missing package/Python command
|
||||
and compile-time DTO/Duration/Drive-error mistakes also prove no product behavior.
|
||||
|
||||
### Explicit limits, not an expanding audit gate
|
||||
|
||||
The four lower-edge rows above separate confirmed direct placement repairs from
|
||||
Workdir create / external grant cleanup / provider acknowledgement continuation
|
||||
questions and catalog producer quiescence. The latter are not redesigned here,
|
||||
per the user's instruction not to expand speculative Workdir/grant or unrelated
|
||||
Feature work indefinitely. Their existing durable recovery, explicit cleanup
|
||||
retry and provider cancellation policy are preserved, not reclassified as
|
||||
end-to-end cancellation proof. This report is not a mechanically complete
|
||||
transitive call graph or an assertion that every future/composed caller is safe.
|
||||
T743's separate new API composition still requires its owner's exact combined
|
||||
source validation/review. None of these limits constitutes Ticket conclusion,
|
||||
MR approval, target integration or permission to update the running environment.
|
||||
|
||||
### Post-suite focused validation blocker
|
||||
|
||||
After the successful full suite above, the current candidate was checked with
|
||||
`cargo test -p yoi-workspace-server --lib db_execution_tests` using the same
|
||||
candidate target directory and ordinary parallel execution. Result: **31 passed /
|
||||
1 failed**, in 8.06s. Failure:
|
||||
`cancelled_runtime_key_revoke_commits_audit_and_removes_broker_registration`
|
||||
reported the shared `observed` helper's two-second `Elapsed(())` timeout, without
|
||||
identifying which of its several lifecycle waits timed out. Source inspection
|
||||
confirms projection shutdown is already present in this fixture; it is not
|
||||
legitimate to attribute the failure to omitted projection shutdown, treat it as
|
||||
load-only, increase limits, reduce parallelism, or rerun until green.
|
||||
|
||||
Evidence is preserved in `target/t748-caller-final-boundaries.log`. The prior
|
||||
full-suite success is real but insufficient to declare final validation complete.
|
||||
The new caller and stop cases all passed in this focused run. Next diagnosis must
|
||||
identify the exact wait and inspect actual HTTP/job/broker ownership events before
|
||||
choosing a fixture or product repair. No normal push, fresh review, approval,
|
||||
integration or Ticket conclusion was performed for this candidate.
|
||||
|
||||
## Continued diagnosis and final validation
|
||||
|
||||
Returning the failed test as a stop condition was a workflow mistake, not an
|
||||
external outage, lost authority, or additional audit approval gate. Work continued
|
||||
on the same branch. The failed run above is retained as evidence.
|
||||
|
||||
The `observed` helper now reports its caller location. A diagnostic 32-case run
|
||||
passed; that result alone was **not** treated as a diagnosis or used to erase the
|
||||
failure. Source tracing identified an uncontrolled cold broker signing producer
|
||||
in the revoke fixture. The test now receives the actual signed WebSocket
|
||||
handshake and drains its completed DB work **before** holding the revoke writer.
|
||||
The handshake remains pending, so broker shutdown is still tested. An early HTTP
|
||||
response is selected against actual SQLite entry, reporting status/body rather
|
||||
than waiting for an event that can no longer occur. The original failed run did
|
||||
not identify its exact wait; attributing it conclusively to signing would exceed
|
||||
the captured evidence.
|
||||
|
||||
Subsequent full runs exposed the Ready table's distinct failure **before** source
|
||||
POST arrival (caller location, not timer/control failure). Artifacts:
|
||||
`target/t748-post-fixture-server-suite.log` and
|
||||
`target/t748-final-framing-server-suite.log`. The HTTP fake previously assumed
|
||||
one TCP read contained the request line and answered without consuming a body.
|
||||
It now parses the complete header/body; a deterministic two-byte-fragment reader
|
||||
proves complete frame consumption and truncated-body rejection. This is a real
|
||||
fixture flaw but does not by itself establish the historical timeout cause.
|
||||
|
||||
Ready setup includes mutation and best-effort notification before the source
|
||||
POST. The held-provider arrival now uses the existing five-second fixture
|
||||
watchdog as its setup bound, selected against handler completion. The subsequent
|
||||
**timer and independent SQL/control observations retain their two-second budget**;
|
||||
actual held-request cancellation/lease/overload/cleanup assertions are unchanged.
|
||||
Production admission limits, normal test parallelism, and provider timeout are
|
||||
unchanged. This setup-bound change is explicit, not a hidden limit increase or
|
||||
claim that Ready completes within two seconds under a whole-suite workload.
|
||||
Restoring the exact published old Ready method still fails this revised test
|
||||
(`target/t748-ready-old-boundary-red.log`, Ready returns only after synchronous
|
||||
provider wait); restored code passes, so the offload distinction is not weakened.
|
||||
|
||||
Final code validation after these fixture changes:
|
||||
- `target/t748-boundary-setup-fixed-green.log`: **33 DB-boundary cases pass**.
|
||||
- `target/t748-final-setup-root-check.log`: root compile closure passes.
|
||||
- `target/t748-final-setup-server-suite.log`: original default-parallel Server
|
||||
**989 passed / 0 failed / 1 existing ignored**, **11 binary tests**, 189.66s.
|
||||
- `target/t748-final-setup-workdir-default.log`: **153 pass** across targets.
|
||||
- `target/t748-final-setup-workdir-http.log`: **156 pass** across targets.
|
||||
- `cargo fmt --all -- --check`, `git diff --check HEAD`, and state-contract names
|
||||
**558 / 0 unapproved / 0 stale** pass.
|
||||
|
||||
No failed semantic run was merely repeated until green: the intervening source
|
||||
changes and the limits of each diagnosis are recorded above. These results support
|
||||
publication for fresh exact-source independent review, not reviewer approval,
|
||||
MR integration, Ticket conclusion, or updates to the live dogfooding environment.
|
||||
@@ -430,8 +430,12 @@ panic diagnostics appeared, and no retained test/repro processes were observed.
|
||||
Workdir default153/http-client156, fmt/diff and naming558/0/0 passed. All Cargo
|
||||
checks/tests use the isolated candidate cache. Logs:
|
||||
`target/t748-revoke-{root-check,full-suite,workdir-default,workdir-http,fmt,names}.log`.
|
||||
The following documentation-only publication changes no tested code and needs a
|
||||
fresh exact-source independent verdict before an approved handoff.
|
||||
Docs-only source `ebce7ab7` changed no tested code. MR seq13 independently confirmed
|
||||
the revoke ownership fix and repeated the full checks, but requested changes for
|
||||
Ticket cancellation, selector repair and post-restore projection's async callers.
|
||||
The current uncommitted follow-up and the requested comprehensive caller audit
|
||||
are tracked in [the caller inventory](2026-10-12-t748-caller-inventory.md).
|
||||
Successful suites are not evidence that uninspected caller boundaries are safe.
|
||||
|
||||
No latency benchmark, production incident measurement, target integration,
|
||||
Ticket conclusion, or running-environment update is claimed. T743 has now
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# Internal SubWorker follow-up observation gap
|
||||
|
||||
## Observed barrier
|
||||
|
||||
During T748's read-only caller inventory, the parent could read the first Runtime
|
||||
inventory assistant result by stable entry ref
|
||||
`E01a12e76-3893-70a0-bb73-28ee0c52f87b`. It then sent a follow-up to the same
|
||||
Idle direct child, `t748-runtime-caller-inventory`, to close four remaining
|
||||
execution/ownership boundaries.
|
||||
|
||||
After a turn-ended/Idle notification, `WorkerList` confirmed Idle, but bounded
|
||||
`SearchSessionEntries` for assistant entries after follow-up user ref
|
||||
`E01a12e76-d696-73d3-80d7-b8cde9935b37` returned no entries. Tool observations
|
||||
were visible. The overview showed only the initial and first follow-up user
|
||||
anchors, not a final result. A single additional Submit asking for the existing
|
||||
findings (not re-investigation) returned success. After its turn-ended/Idle
|
||||
notification, the same bounded assistant query still returned no entries, and the
|
||||
overview still ended at the first follow-up user anchor.
|
||||
|
||||
This is an observation-surface symptom, **not proof** that an assistant result
|
||||
was generated, lost, or that the underlying investigation completed. No raw
|
||||
session storage or hidden worker state was read to bypass the projection.
|
||||
|
||||
The same symptom also occurred for `t748-domain-caller-inventory`: its first
|
||||
result `E01a12e73-7ae4-7f02-9a7e-203140971635` was readable, but after the
|
||||
follow-up turn-ended/Idle notification the overview ended at the follow-up's
|
||||
starting commentary `E01a12e75-9c19-7bd0-9284-299116a05761` and intervening tool
|
||||
records. No final follow-up result was visible. The parent released that
|
||||
read-only delegation without another Submit and took over the remaining generated
|
||||
adapter/forwarder and launch lower-edge checks. Neither `WorkerStop` tool outcome
|
||||
was treated as evidence that the investigation completed.
|
||||
|
||||
## Handling
|
||||
|
||||
The parent did not mark the delegated work or Ticket complete, publish source,
|
||||
or replace independent review with the notification. It stopped only the two
|
||||
parent-owned read-only children to release delegation and took over the open
|
||||
Runtime and domain boundaries. No live product process, parent Worker, target
|
||||
branch, or attachment was stopped or cleaned.
|
||||
|
||||
## Improvement proposal
|
||||
|
||||
Expose enough committed turn metadata through the authorized observation surface
|
||||
to distinguish a finalized assistant result, a turn ending without a result, and
|
||||
an observation capture not yet including an accepted Submit. An explicit
|
||||
completion marker tied to the visible Submit entry would prevent an Idle
|
||||
notification from being confused with inspectable evidence. Preserve append-only
|
||||
session entries and the current authorization projection; do not expose raw
|
||||
prompts/reasoning or require polling/retry loops as a workaround.
|
||||
Reference in New Issue
Block a user