feat(worker): separate restored candidates, repair, and activation
This commit is contained in:
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
/// Open a current retained Session without creating or migrating anything.
|
||||
/// The live constraint is checked here and again by every later publication;
|
||||
/// attaching the control store itself does not acquire publication authority.
|
||||
pub fn open_existing_with_retention(
|
||||
root: impl Into<PathBuf>,
|
||||
retention: crate::retention::SessionRetentionStore,
|
||||
) -> Result<Self, StoreError> {
|
||||
let mut store = Self::open_read_only(root)?;
|
||||
if let Some(session) = *store.session_id.lock().unwrap() {
|
||||
retention.ensure_live(session)?;
|
||||
}
|
||||
store.retention = Some(retention);
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
fn lock_publication(
|
||||
&self,
|
||||
session: SessionId,
|
||||
|
||||
@@ -2301,9 +2301,28 @@ impl WorkerAggregateRecord {
|
||||
});
|
||||
}
|
||||
if let Some(request) = request {
|
||||
let mut validation_request = request.clone();
|
||||
// Pending admission may supply a missing Workspace connection
|
||||
// without applying it to the saved execution. Validate that
|
||||
// exact owner's intent against identity; do not widen the
|
||||
// Available-record rule or mutate the canonical request.
|
||||
if validation_request.workspace_api.is_none()
|
||||
&& let PersistedWorkerLifecycleOperation::Restore(restore) = &operation
|
||||
&& let Some(api) = restore_guard
|
||||
.pending_preparation(restore.operation_id)
|
||||
.and_then(|preparation| preparation.workspace_api.as_ref())
|
||||
{
|
||||
validation_request.workspace_api =
|
||||
Some(
|
||||
serde_json::from_value(serde_json::to_value(api).map_err(
|
||||
|error| runtime_store_corrupt(path, error.to_string()),
|
||||
)?)
|
||||
.map_err(|error| runtime_store_corrupt(path, error.to_string()))?,
|
||||
);
|
||||
}
|
||||
WorkerExecutionRecord {
|
||||
schema_version: SCHEMA_VERSION,
|
||||
request: request.clone(),
|
||||
request: validation_request,
|
||||
binding: binding.clone(),
|
||||
restore_intent,
|
||||
}
|
||||
|
||||
@@ -16,6 +16,9 @@ pub mod management;
|
||||
pub mod observation;
|
||||
pub mod profile_archive;
|
||||
pub mod resource;
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
#[cfg(feature = "fs-store")]
|
||||
pub mod retention;
|
||||
mod runtime;
|
||||
|
||||
@@ -682,8 +682,10 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
// The same TempDir owns the relative bind/connect reference. No cwd change
|
||||
// or alias is needed, and the real live-socket migration refusal stays tested.
|
||||
let temp = tempfile::tempdir_in(".").unwrap();
|
||||
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
||||
let before = tree_bytes(&root);
|
||||
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,173 @@
|
||||
//! Explicit, operation-owned repair of saved Session material. This is not
|
||||
//! read-only candidate preparation: the canonical Session/attribution is the
|
||||
//! receipt for replay after a lost response, and effects are reported as such.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl ProfileRuntimeWorkerFactory {
|
||||
pub(super) async fn recover_saved_restore_source(
|
||||
&self,
|
||||
request: &WorkerExecutionRestoreRequest,
|
||||
) -> Result<bool, RuntimeRestoreFailure> {
|
||||
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
|
||||
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
|
||||
let root = self
|
||||
.worker_aggregate_dir(&request.worker_ref)
|
||||
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
|
||||
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
|
||||
.map_err(|_| cause("restore recovery metadata open failed"))?;
|
||||
let metadata = metadata_store
|
||||
.read_by_name(&name)
|
||||
.map_err(|_| cause("restore recovery metadata read failed"))?
|
||||
.ok_or_else(|| cause("restore recovery metadata missing"))?;
|
||||
let active = metadata
|
||||
.active
|
||||
.as_ref()
|
||||
.ok_or_else(|| cause("restore recovery Session missing"))?;
|
||||
let pending = active.segment_id.is_none();
|
||||
let incomplete = request.request.subjektiv_attached
|
||||
&& !matches!(
|
||||
metadata.subjektiv_session_attribution,
|
||||
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
|
||||
);
|
||||
if !pending && !incomplete {
|
||||
return Ok(false);
|
||||
}
|
||||
// The accepted Runtime operation already owns this Session. Its stable
|
||||
// Session identity and sole committed seed, not a new restore request,
|
||||
// are used to reconcile an interrupted first-head publication.
|
||||
let session_store = WorkerSessionStore::new_with_retention(
|
||||
root.join("session"),
|
||||
self.session_retention()
|
||||
.map_err(|_| cause("restore recovery retention unavailable"))?,
|
||||
)
|
||||
.map_err(|_| cause("restore recovery Session open failed"))?;
|
||||
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
|
||||
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
|
||||
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
|
||||
&request.request,
|
||||
self.runtime_id.as_deref(),
|
||||
);
|
||||
let request_client = request
|
||||
.request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
|
||||
let context = workspace.worker_context(
|
||||
&request.worker_ref,
|
||||
request.workspace_scope.as_ref(),
|
||||
self.worker_mutation_identity.as_ref(),
|
||||
request_client,
|
||||
self.embedded_worker_mutation_dispatcher.as_ref(),
|
||||
Some(self.prompt_projection_cache.clone()),
|
||||
);
|
||||
if pending && context.workspace_id().is_some() {
|
||||
let bundle = request
|
||||
.config_bundle
|
||||
.as_ref()
|
||||
.ok_or_else(|| cause("pending recovery launch material missing"))?;
|
||||
let projection = self
|
||||
.observe_bundle_prompt_projection(
|
||||
bundle,
|
||||
request
|
||||
.request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.map(|api| api.workspace_id.as_str()),
|
||||
)
|
||||
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
|
||||
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
|
||||
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
|
||||
}
|
||||
self.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(
|
||||
request.worker_ref.clone(),
|
||||
FailedRestoreResources {
|
||||
operation_id: request.operation_id,
|
||||
sessions: Vec::new(),
|
||||
candidate_workdir_cleanup: None,
|
||||
allocation: None,
|
||||
feature_tasks: None,
|
||||
},
|
||||
);
|
||||
let store = CombinedStore::new(session_store, metadata_store);
|
||||
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
|
||||
&name,
|
||||
manifest,
|
||||
store,
|
||||
loader,
|
||||
context,
|
||||
WorkerFilesystemAuthority::None,
|
||||
)
|
||||
.await;
|
||||
let mut worker = worker_result.map_err(|error| {
|
||||
self.retain_construction_failure(request, &error);
|
||||
cause(restore_worker_cause(&error))
|
||||
})?;
|
||||
let tasks = worker.retain_startup_feature_cleanup();
|
||||
let sessions = worker.workdir_sessions();
|
||||
let allocation = worker.restore_allocation_cleanup_handle();
|
||||
self.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(
|
||||
request.worker_ref.clone(),
|
||||
FailedRestoreResources {
|
||||
operation_id: request.operation_id,
|
||||
sessions: vec![sessions.clone()],
|
||||
candidate_workdir_cleanup: None,
|
||||
allocation: allocation.clone(),
|
||||
feature_tasks: Some(tasks.clone()),
|
||||
},
|
||||
);
|
||||
let result = async {
|
||||
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||
worker
|
||||
.bind_backend_job(binding.clone())
|
||||
.map_err(|_| cause("recovery Backend Job binding failed"))?;
|
||||
}
|
||||
worker
|
||||
.finalize_subjektiv_session_attribution(
|
||||
request.request.subjektiv_attached,
|
||||
SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||
)
|
||||
.map_err(|error| cause(restore_worker_cause(&error)))?;
|
||||
if pending {
|
||||
worker::WorkerController::recover_pending_head(&mut worker, &root)
|
||||
.await
|
||||
.map_err(|_| cause("pending Session head recovery failed"))?;
|
||||
}
|
||||
Ok::<_, RuntimeRestoreFailure>(())
|
||||
}
|
||||
.await;
|
||||
// Never release allocation while the Worker guard is attached. Retained
|
||||
// handles in the same operation remain available on every cleanup error.
|
||||
drop(worker);
|
||||
let cleanup = async {
|
||||
tasks
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|_| cause("recovery Feature settlement failed"))?;
|
||||
sessions
|
||||
.close_all()
|
||||
.await
|
||||
.map_err(|_| cause("recovery Workdir close failed"))?;
|
||||
if let Some(allocation) = allocation {
|
||||
allocation
|
||||
.release()
|
||||
.map_err(|_| cause("recovery allocation release failed"))?;
|
||||
}
|
||||
Ok::<_, RuntimeRestoreFailure>(())
|
||||
}
|
||||
.await;
|
||||
result?;
|
||||
cleanup?;
|
||||
self.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&request.worker_ref);
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,898 @@
|
||||
// Included inside worker_backend::tests, alongside the restore failure suite.
|
||||
// These integration tests connect the real Profile factory, canonical stores,
|
||||
// restore owner, and Controller. Only model I/O and the Workspace HTTP transport
|
||||
// are substituted. The loopback Host is not an authentication/policy test.
|
||||
// Run in the ordinary worker-runtime lib suite with the default product features
|
||||
// and the filter `restore_candidate::`. No live service is used. Like the
|
||||
// failure suite, this requires the existing Yoi path settings to resolve into
|
||||
// target/restore-test-environment; the common fixture rejects unsafe fallbacks
|
||||
// before any factory, Git, or allocation-table operation.
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
mod restore_candidate {
|
||||
use super::*;
|
||||
|
||||
const FIRST: &str = "candidate-saved-first";
|
||||
const SECOND: &str = "candidate-saved-second";
|
||||
const NOTICE: &str = "candidate-saved-notice";
|
||||
const FRESH: &str = "candidate-fresh-input";
|
||||
const WAIT: Duration = Duration::from_secs(5);
|
||||
|
||||
struct HostProbe {
|
||||
calls: Arc<Mutex<Vec<(String, String)>>>,
|
||||
task: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Drop for HostProbe {
|
||||
fn drop(&mut self) {
|
||||
if let Some(task) = self.task.take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Aborting on unwind prevents a failed setup assertion from leaving its
|
||||
// hanging model turn alive. Successful setup uses the normal shutdown/join.
|
||||
struct SeedController(Option<RuntimeWorkerController>);
|
||||
|
||||
impl Drop for SeedController {
|
||||
fn drop(&mut self) {
|
||||
if let Some(controller) = self.0.as_ref() {
|
||||
controller.controller_task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn receive_matching(
|
||||
events: &mut tokio::sync::broadcast::Receiver<Event>,
|
||||
phase: &str,
|
||||
mut matches: impl FnMut(&Event) -> bool,
|
||||
) -> Event {
|
||||
let mut seen = Vec::new();
|
||||
tokio::time::timeout(WAIT, async {
|
||||
loop {
|
||||
let event = events.recv().await.expect("Controller event stream");
|
||||
seen.push(format!("{event:?}"));
|
||||
if matches(&event) {
|
||||
return event;
|
||||
}
|
||||
assert!(
|
||||
!matches!(
|
||||
event,
|
||||
Event::Error { .. }
|
||||
| Event::SubmissionRejected { .. }
|
||||
| Event::NotificationRejected { .. }
|
||||
),
|
||||
"unexpected Controller rejection: {event:?}"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!("Controller acknowledgement deadline during {phase}; observed {seen:?}")
|
||||
})
|
||||
}
|
||||
|
||||
async fn submit(handle: &WorkerHandle, id: &str) -> Event {
|
||||
let mut events = handle.subscribe();
|
||||
handle
|
||||
.send(Method::Submit {
|
||||
submission_request_id: id.into(),
|
||||
input: vec![protocol::Segment::Text { content: id.into() }],
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
receive_matching(&mut events, id, |event| {
|
||||
matches!(event, Event::SubmissionAccepted { submission_request_id, .. }
|
||||
if submission_request_id == id)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn notify(handle: &WorkerHandle) {
|
||||
let mut events = handle.subscribe();
|
||||
handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: NOTICE.into(),
|
||||
message: NOTICE.into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
receive_matching(&mut events, "notification acceptance", |event| {
|
||||
matches!(event, Event::NotificationAccepted { notification_request_id }
|
||||
if notification_request_id == NOTICE)
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn pending(handle: &WorkerHandle) -> protocol::PendingSubmissionsSnapshot {
|
||||
let mut events = handle.subscribe();
|
||||
handle.send(Method::ListPendingSubmissions).await.unwrap();
|
||||
match receive_matching(&mut events, "pending queue snapshot", |event| {
|
||||
matches!(event, Event::PendingSubmissionsChanged { .. })
|
||||
})
|
||||
.await
|
||||
{
|
||||
Event::PendingSubmissionsChanged { pending } => pending,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn finished(handle: &WorkerHandle, id: &str) {
|
||||
let mut events = handle.subscribe();
|
||||
let is_finished = |snapshot: &protocol::WorkerStateSnapshot| {
|
||||
snapshot.last_finished_submission_request_id.as_deref() == Some(id)
|
||||
&& snapshot.state == protocol::WorkerState::Idle
|
||||
};
|
||||
if !is_finished(&handle.shared_state.snapshot()) {
|
||||
receive_matching(
|
||||
&mut events,
|
||||
id,
|
||||
|event| matches!(event, Event::WorkerState { snapshot } if is_finished(snapshot)),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn queue_checkpoint(entries: &[LogEntry]) -> serde_json::Value {
|
||||
entries
|
||||
.iter()
|
||||
.rev()
|
||||
.find_map(|entry| match entry {
|
||||
LogEntry::Extension {
|
||||
domain, payload, ..
|
||||
} if domain == "worker.pending_activations.v1" => Some(payload.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.expect("real Controller must have committed its pending activation checkpoint")
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
backend: Arc<WorkerRuntimeExecutionBackend>,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
model: MockClient,
|
||||
session: WorkerSessionStore,
|
||||
session_id: session_store::SessionId,
|
||||
segment_id: session_store::SegmentId,
|
||||
receipts: Vec<serde_json::Value>,
|
||||
host: HostProbe,
|
||||
_root: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn new() -> Self {
|
||||
Self::with_transport(WorkerControllerTransport::InProcess)
|
||||
}
|
||||
|
||||
fn with_transport(transport: WorkerControllerTransport) -> Self {
|
||||
let root = restore_failure_tempdir();
|
||||
let model = MockClient::sequential(vec![
|
||||
MockResponse::Complete(vec![
|
||||
LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"),
|
||||
LlmEvent::tool_input_delta(
|
||||
0,
|
||||
r#"{"subject":"Activated candidate","description":"Execution probe"}"#,
|
||||
),
|
||||
LlmEvent::tool_use_stop(0),
|
||||
LlmEvent::Status(StatusEvent {
|
||||
status: ResponseStatus::Completed,
|
||||
}),
|
||||
]),
|
||||
MockResponse::Complete(simple_text_events()),
|
||||
MockResponse::Complete(simple_text_events()),
|
||||
MockResponse::Complete(simple_text_events()),
|
||||
]);
|
||||
let injected = model.clone();
|
||||
let mut factory = ProfileRuntimeWorkerFactory::new(root.path())
|
||||
.with_runtime_store_dir(root.path())
|
||||
.with_runtime_id("candidate-runtime")
|
||||
.with_spawn_model_client_factory(move || Box::new(injected.clone()));
|
||||
factory.controller_transport = transport;
|
||||
let catalog = worker::PromptCatalog::builtins_only()
|
||||
.unwrap()
|
||||
.projection()
|
||||
.clone();
|
||||
factory
|
||||
.observe_workspace_prompt_projection(
|
||||
worker::WorkspacePromptProjection::new(
|
||||
"workspace-saved-subjektiv",
|
||||
"candidate-fixture",
|
||||
catalog.catalog_digest.clone(),
|
||||
catalog,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
|
||||
let fixture_root = root.path().to_path_buf();
|
||||
let (request, session, session_id, segment_id, receipts, host) = backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let authorization_calls = calls.clone();
|
||||
let other_calls = calls.clone();
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let authorization_path =
|
||||
"/api/w/workspace-saved-subjektiv/authorization/check";
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
authorization_path,
|
||||
axum::routing::post(move |axum::Json(input): axum::Json<
|
||||
server_api::authorization::OperationAuthorizationRequest,
|
||||
>| {
|
||||
let calls = authorization_calls.clone();
|
||||
async move {
|
||||
calls.lock().unwrap().push((
|
||||
"POST".into(), authorization_path.into(),
|
||||
));
|
||||
let response = if input.operation == "task"
|
||||
&& input.resource == server_api::authorization::PermissionResource::Feature
|
||||
{
|
||||
server_api::authorization::OperationAuthorizationResponse {
|
||||
scope: Some(server_api::authorization::PermissionResource::Feature),
|
||||
}
|
||||
} else {
|
||||
saved_subjektiv_authorization_fixture(input)
|
||||
.expect("only the saved fixture Feature grants are available")
|
||||
};
|
||||
axum::Json(response)
|
||||
}
|
||||
}),
|
||||
)
|
||||
.fallback(move |request: axum::extract::Request| {
|
||||
let calls = other_calls.clone();
|
||||
async move {
|
||||
calls.lock().unwrap().push((
|
||||
request.method().to_string(), request.uri().path().to_owned(),
|
||||
));
|
||||
axum::http::StatusCode::FORBIDDEN
|
||||
}
|
||||
});
|
||||
let host = HostProbe {
|
||||
calls,
|
||||
task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })),
|
||||
};
|
||||
let worker_ref = WorkerRef::new(WorkerId::now_v7());
|
||||
let (mut manifest, mut create) =
|
||||
saved_subjektiv_policy(&fixture_root, &worker_ref);
|
||||
// Normal restore deliberately excludes Subject repair. Keep
|
||||
// a real Workspace client so accidental Host effects are visible.
|
||||
manifest.feature.subjektiv = manifest::ResolvedSubjektivFeatureConfig::default();
|
||||
manifest.feature.task.enabled = true;
|
||||
create.subjektiv_attached = false;
|
||||
create.workspace_api.as_mut().unwrap().base_url = base_url;
|
||||
let aggregate = fixture_root.join("workers").join(worker_ref.worker_id.to_string());
|
||||
let session = WorkerSessionStore::new(aggregate.join("session")).unwrap();
|
||||
let metadata = WorkerAggregateStore::new(&aggregate, &manifest.worker.name).unwrap();
|
||||
let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||
MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]),
|
||||
);
|
||||
engine.set_system_prompt("Committed restore candidate fixture prompt");
|
||||
let mut worker = Worker::new(
|
||||
manifest.clone(),
|
||||
engine,
|
||||
CombinedStore::new(session.clone(), metadata.clone()),
|
||||
RuntimeWorkspaceBackendRef::from_worker_request(&create, Some("candidate-runtime"))
|
||||
.worker_context(&worker_ref, Some(&crate::runtime::RuntimeWorkspaceScope::new(
|
||||
"workspace-saved-subjektiv", "server-main",
|
||||
)), None, None, None, None),
|
||||
WorkerFilesystemAuthority::None,
|
||||
Scope::empty(),
|
||||
).await.unwrap();
|
||||
worker.enable_worker_metadata_write_through().unwrap();
|
||||
worker.materialize_durable_session_head().await.unwrap();
|
||||
let (session_id, segment_id) = (worker.session_id(), worker.segment_id());
|
||||
let workspace_client = worker.workspace_client_handle();
|
||||
let started = PreparedWorker::new(
|
||||
worker,
|
||||
WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") },
|
||||
WorkerControllerTransport::InProcess,
|
||||
).start().await.unwrap();
|
||||
let mut seed = SeedController(Some(RuntimeWorkerController {
|
||||
handle: started.handle,
|
||||
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||
controller_task: started.controller_task,
|
||||
workspace_client,
|
||||
}));
|
||||
let handle = seed.0.as_ref().unwrap().handle.clone();
|
||||
let mut started_events = handle.subscribe();
|
||||
submit(&handle, "candidate-setup-turn").await;
|
||||
// Running and streaming observations can arrive in either
|
||||
// order. Do not discard TextDelta while awaiting WorkerState.
|
||||
let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running;
|
||||
let mut streaming = false;
|
||||
receive_matching(&mut started_events, "seed running and streaming", |event| {
|
||||
if let Event::WorkerState { snapshot } = event {
|
||||
running |= snapshot.catalog_status() == protocol::WorkerStatus::Running;
|
||||
}
|
||||
streaming |= matches!(event, Event::TextDelta { .. });
|
||||
running && streaming
|
||||
}).await;
|
||||
let state = Arc::new(RwLock::new(handle.shared_state.snapshot()));
|
||||
let mut events = handle.subscribe();
|
||||
handle.send(Method::Pause { command: next_internal_command(&state).unwrap() })
|
||||
.await.unwrap();
|
||||
receive_matching(&mut events, "seed pause acknowledgement", |event| {
|
||||
matches!(event, Event::WorkerState { snapshot }
|
||||
if snapshot.catalog_status() == protocol::WorkerStatus::Paused)
|
||||
}).await;
|
||||
let mut receipts = Vec::new();
|
||||
for id in [FIRST, SECOND] {
|
||||
let receipt = submit(&handle, id).await;
|
||||
assert!(matches!(receipt, Event::SubmissionAccepted {
|
||||
disposition: protocol::SubmissionDisposition::Queued, ..
|
||||
}));
|
||||
receipts.push(serde_json::to_value(receipt).unwrap());
|
||||
if id == FIRST { notify(&handle).await; }
|
||||
}
|
||||
assert_eq!(pending(&handle).await.submissions.len(), 2);
|
||||
shutdown_profile_controller(seed.0.take().unwrap()).await;
|
||||
// This orphan pin would be removed by the normal activation
|
||||
// reconciliation. Preparation and discard must leave it alone.
|
||||
let orphan = session.write_uploaded_file(
|
||||
session_id, "orphan.txt", "text/plain", b"retained orphan",
|
||||
session_store::UploadedFileLimits::default(),
|
||||
).unwrap();
|
||||
session.pin_uploaded_file(session_id, &orphan, "uncommitted-acceptance").unwrap();
|
||||
let mut saved = metadata.read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref))
|
||||
.unwrap().unwrap();
|
||||
saved.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
|
||||
saved.workspace_root = Some(fixture_root.join("stale-derived-root"));
|
||||
metadata.write(&saved).unwrap();
|
||||
let mut request = direct_restore_request(worker_ref);
|
||||
request.request = create;
|
||||
request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new(
|
||||
"workspace-saved-subjektiv", "server-main",
|
||||
));
|
||||
Ok((request, session, session_id, segment_id, receipts, host))
|
||||
})
|
||||
.unwrap();
|
||||
let fixture = Self {
|
||||
backend,
|
||||
request,
|
||||
model,
|
||||
session,
|
||||
session_id,
|
||||
segment_id,
|
||||
receipts,
|
||||
host,
|
||||
_root: root,
|
||||
};
|
||||
let checkpoint = queue_checkpoint(&fixture.entries());
|
||||
assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2);
|
||||
assert_eq!(
|
||||
checkpoint["pending_notifications"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
fixture.assert_quiet();
|
||||
fixture
|
||||
}
|
||||
|
||||
fn entries(&self) -> Vec<LogEntry> {
|
||||
self.session
|
||||
.read_all(self.session_id, self.segment_id)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn canonical_files(&self) -> BTreeMap<PathBuf, Vec<u8>> {
|
||||
let aggregate = self
|
||||
.backend
|
||||
.factory
|
||||
.worker_aggregate_dir(&self.request.worker_ref)
|
||||
.unwrap();
|
||||
let mut files = persisted_files(&aggregate);
|
||||
files.retain(|path, _| !path.components().any(|part| part.as_os_str() == "runs"));
|
||||
files
|
||||
}
|
||||
|
||||
fn assert_no_modifying_host_calls(&self) {
|
||||
assert!(
|
||||
self.host
|
||||
.calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|(method, path)| {
|
||||
method == "GET"
|
||||
|| method == "HEAD"
|
||||
|| (method == "POST"
|
||||
&& path == "/api/w/workspace-saved-subjektiv/authorization/check")
|
||||
}),
|
||||
"unexpected modifying Host call"
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_quiet(&self) {
|
||||
assert_eq!(self.model.call_count.load(Ordering::SeqCst), 0);
|
||||
assert!(self.model.captured.lock().unwrap().is_empty());
|
||||
self.assert_no_modifying_host_calls();
|
||||
assert!(
|
||||
session_store::collect_state(&self.entries())
|
||||
.annotated_history
|
||||
.iter()
|
||||
.all(|entry| {
|
||||
!matches!(
|
||||
entry.item,
|
||||
LoggedItem::ToolCall { .. } | LoggedItem::ToolResult { .. }
|
||||
)
|
||||
}),
|
||||
"prepare committed tool execution"
|
||||
);
|
||||
}
|
||||
|
||||
fn prepare(&self) {
|
||||
assert!(matches!(
|
||||
self.backend.restore_worker(self.request.clone()),
|
||||
WorkerExecutionSpawnResult::Connected { .. }
|
||||
));
|
||||
// Connected here is the backend's prepared snapshot, not publication.
|
||||
assert!(
|
||||
self.backend
|
||||
.pending_restore(&self.request.worker_ref)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||
assert!(
|
||||
self.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.get(&self.request.worker_ref)
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
!self
|
||||
.backend
|
||||
.factory
|
||||
.worker_restore_run_dir(&self.request.worker_ref, self.request.operation_id,)
|
||||
.unwrap()
|
||||
.exists()
|
||||
);
|
||||
let table = worker::runtime::worker_allocation::LockFileGuard::open(
|
||||
&worker::runtime::worker_allocation::default_allocation_path().unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
table
|
||||
.data()
|
||||
.find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(
|
||||
&self.request.worker_ref,
|
||||
))
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_no_allocation(&self) {
|
||||
let table = worker::runtime::worker_allocation::LockFileGuard::open(
|
||||
&worker::runtime::worker_allocation::default_allocation_path().unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
table
|
||||
.data()
|
||||
.find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(
|
||||
&self.request.worker_ref,
|
||||
))
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_released(&self) {
|
||||
self.assert_no_allocation();
|
||||
assert!(
|
||||
self.backend
|
||||
.pending_restore(&self.request.worker_ref)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||
assert!(
|
||||
self.backend
|
||||
.factory
|
||||
.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
self.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.get(&self.request.worker_ref)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
// Keep the storage and loopback transport alive until owned tasks stop.
|
||||
let _ = self.backend.stop_worker(&self.request.worker_ref);
|
||||
if let Some(task) = self.host.task.take() {
|
||||
task.abort();
|
||||
let _ = self.backend.run_on_adapter_runtime(async move {
|
||||
let _ = task.await;
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn prepare_saved_inputs_rejects_new_input_and_discard_preserves_authority() {
|
||||
// Exercise the configured socket transport without ever publishing or
|
||||
// binding it; activation coverage below uses the normal InProcess path.
|
||||
let mut fixture = Fixture::with_transport(WorkerControllerTransport::UnixSocket);
|
||||
let opens = Arc::new(AtomicUsize::new(0));
|
||||
let counted = opens.clone();
|
||||
let raw = deferred_workdir::DeferredWorkdirSession::handle(
|
||||
"candidate-discard-resource",
|
||||
workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||
move || {
|
||||
counted.fetch_add(1, Ordering::SeqCst);
|
||||
Err("not opened during prepare".into())
|
||||
},
|
||||
);
|
||||
fixture.request.deferred_workdir_sessions.insert(
|
||||
WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||
raw.clone(),
|
||||
);
|
||||
let before = fixture.canonical_files();
|
||||
fixture.prepare();
|
||||
for input in [
|
||||
WorkerInput::user("must-not-accept"),
|
||||
WorkerInput::notify("must-not-notify"),
|
||||
] {
|
||||
assert!(
|
||||
!fixture
|
||||
.backend
|
||||
.dispatch_input(&fixture.request.worker_ref, input)
|
||||
.is_accepted()
|
||||
);
|
||||
}
|
||||
#[cfg(feature = "ws-server")]
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.clone()
|
||||
.attach_worker_protocol(&fixture.request.worker_ref)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!matches!(
|
||||
fixture
|
||||
.backend
|
||||
.worker_session(WorkerSessionObservationRequest {
|
||||
worker_ref: fixture.request.worker_ref.clone(),
|
||||
}),
|
||||
runtime_api::WorkerSessionAvailability::LiveProtocol
|
||||
));
|
||||
fixture.assert_quiet();
|
||||
assert_eq!(fixture.canonical_files(), before);
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.factory
|
||||
.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap()[&fixture.request.worker_ref]
|
||||
.candidate_workdir_cleanup
|
||||
.is_some(),
|
||||
"retain the real prepared Worker's close-only resource owner"
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.stop_worker(&fixture.request.worker_ref)
|
||||
.is_accepted()
|
||||
);
|
||||
fixture.assert_released();
|
||||
|
||||
let factory = fixture.backend.factory.clone();
|
||||
let request = fixture.request.clone();
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
// The adapter resolves and owns its own raw handles. Separately
|
||||
// exercise the factory seam with the exact supplied cleanup handle.
|
||||
let mut candidate = factory
|
||||
.prepare_restore(request)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
candidate
|
||||
.discard()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
assert!(matches!(
|
||||
raw.read(workdir::ReadRequest {
|
||||
path: workdir::WorkdirPath::new("probe.txt").unwrap(),
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
max_bytes: 32,
|
||||
})
|
||||
.await,
|
||||
Err(workdir::WorkdirError::SessionClosed)
|
||||
));
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(opens.load(Ordering::SeqCst), 0);
|
||||
fixture.assert_quiet();
|
||||
assert_eq!(fixture.canonical_files(), before);
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.stop_worker(&fixture.request.worker_ref)
|
||||
.is_accepted()
|
||||
);
|
||||
assert_eq!(fixture.canonical_files(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn prepare_failure_at_each_pre_activation_checkpoint_preserves_saved_authority() {
|
||||
// Feature and Controller are activation checkpoints, not preparation.
|
||||
for stage in [
|
||||
RuntimeRestoreStage::Metadata,
|
||||
RuntimeRestoreStage::Reconstruction,
|
||||
RuntimeRestoreStage::Binding,
|
||||
] {
|
||||
let fixture = Fixture::new();
|
||||
let before = fixture.canonical_files();
|
||||
*fixture
|
||||
.backend
|
||||
.factory
|
||||
.restore_faults
|
||||
.fail_at
|
||||
.lock()
|
||||
.unwrap() = Some(stage);
|
||||
let result = fixture.backend.restore_worker(fixture.request.clone());
|
||||
assert!(
|
||||
matches!(result, WorkerExecutionSpawnResult::RolledBack(_)),
|
||||
"{stage:?}: {result:?}"
|
||||
);
|
||||
fixture.assert_released();
|
||||
fixture.assert_quiet();
|
||||
assert_eq!(fixture.canonical_files(), before, "{stage:?}");
|
||||
let completed = fixture.backend.completed_restore_failures.lock().unwrap();
|
||||
let (_, failure) = &completed[&fixture.request.worker_ref];
|
||||
assert_eq!(failure.stage, stage);
|
||||
assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn activate_saved_inputs_drains_fifo_once_and_replays_durable_receipts() {
|
||||
let fixture = Fixture::new();
|
||||
let before = fixture.canonical_files();
|
||||
fixture.prepare();
|
||||
fixture.assert_quiet();
|
||||
assert_eq!(fixture.canonical_files(), before);
|
||||
fixture
|
||||
.backend
|
||||
.activate_restored_worker(fixture.request.operation_id, &fixture.request.worker_ref)
|
||||
.unwrap();
|
||||
let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref]
|
||||
.handle
|
||||
.clone();
|
||||
let model = fixture.model.clone();
|
||||
let receipts = fixture.receipts.clone();
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
finished(&handle, SECOND).await;
|
||||
assert_eq!(model.call_count.load(Ordering::SeqCst), 3);
|
||||
{
|
||||
let requests = model.captured.lock().unwrap();
|
||||
let first = serde_json::to_string(&requests[0].items).unwrap();
|
||||
let second = serde_json::to_string(&requests[2].items).unwrap();
|
||||
assert!(
|
||||
requests[0]
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "TaskCreate")
|
||||
);
|
||||
assert!(first.contains(FIRST));
|
||||
assert!(!first.contains(SECOND), "second Submit overtook the first");
|
||||
assert_eq!(first.matches(NOTICE).count(), 1);
|
||||
assert!(second.find(FIRST).unwrap() < second.find(SECOND).unwrap());
|
||||
assert_eq!(second.matches(NOTICE).count(), 1);
|
||||
}
|
||||
let committed = serde_json::to_value(handle.committed_entries()).unwrap();
|
||||
for (id, mut expected) in [FIRST, SECOND].into_iter().zip(receipts) {
|
||||
// Activation advances the durable receipt from Queued to Started;
|
||||
// replay must preserve its original submission identity.
|
||||
expected["data"]["disposition"] = serde_json::json!("started");
|
||||
assert_eq!(
|
||||
serde_json::to_value(submit(&handle, id).await).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
notify(&handle).await;
|
||||
assert!(pending(&handle).await.submissions.is_empty());
|
||||
assert_eq!(model.call_count.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(
|
||||
serde_json::to_value(handle.committed_entries()).unwrap(),
|
||||
committed
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let history = session_store::collect_state(&fixture.entries()).annotated_history;
|
||||
assert!(
|
||||
history.iter().any(|entry| matches!(
|
||||
&entry.item,
|
||||
LoggedItem::ToolResult { call_id, is_error: false, .. }
|
||||
if call_id == "candidate-task-probe"
|
||||
)),
|
||||
"activation must execute the real TaskCreate tool, not just decode a call"
|
||||
);
|
||||
for marker in [FIRST, SECOND, NOTICE] {
|
||||
let count = history
|
||||
.iter()
|
||||
.filter(|entry| {
|
||||
match &entry.item {
|
||||
LoggedItem::Message { content, .. } => content.iter().any(|part| {
|
||||
matches!(part, LoggedContentPart::Text { text } if text.contains(marker))
|
||||
}),
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
.count();
|
||||
assert_eq!(count, 1, "{marker} must commit exactly once");
|
||||
}
|
||||
let mut fresh = WorkerInput::user(FRESH);
|
||||
fresh.submission_request_id = Some(FRESH.into());
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.dispatch_input(&fixture.request.worker_ref, fresh)
|
||||
.is_accepted()
|
||||
);
|
||||
let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref]
|
||||
.handle
|
||||
.clone();
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
finished(&handle, FRESH).await;
|
||||
assert!(pending(&handle).await.submissions.is_empty());
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(fixture.model.call_count.load(Ordering::SeqCst), 4);
|
||||
assert!(
|
||||
!fixture.host.calls.lock().unwrap().is_empty(),
|
||||
"activation must reach the configured Host transport"
|
||||
);
|
||||
fixture.assert_no_modifying_host_calls();
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.stop_worker(&fixture.request.worker_ref)
|
||||
.is_accepted()
|
||||
);
|
||||
fixture.assert_released();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn explicit_repair_reconciles_lost_attribution_response_without_reseeding() {
|
||||
for materialized in [true, false] {
|
||||
let root = restore_failure_tempdir();
|
||||
let model = MockClient::new(Vec::new());
|
||||
let injected = model.clone();
|
||||
let backend = WorkerRuntimeExecutionBackend::new(
|
||||
ProfileRuntimeWorkerFactory::new(root.path())
|
||||
.with_runtime_store_dir(root.path())
|
||||
.with_runtime_id("repair-runtime")
|
||||
.with_spawn_model_client_factory(move || Box::new(injected.clone())),
|
||||
)
|
||||
.unwrap();
|
||||
let factory = backend.factory.clone();
|
||||
let fixture_root = root.path().to_path_buf();
|
||||
backend.run_on_adapter_runtime(async move {
|
||||
let mut request = saved_subjektiv_restore_fixture(&fixture_root, materialized).await;
|
||||
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref);
|
||||
let aggregate = fixture_root.join("workers").join(request.worker_ref.worker_id.to_string());
|
||||
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
|
||||
let mut saved = metadata.read_by_name(&name).unwrap().unwrap();
|
||||
let session_id = saved.active.as_ref().unwrap().session_id;
|
||||
saved.subjektiv_session_attribution = Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id, diagnostic: "saved interrupted attribution".into() });
|
||||
metadata.write(&saved).unwrap();
|
||||
// Explicit repair may acquire artifact locks; compare canonical
|
||||
// contents separately from that lock-file side effect.
|
||||
let session_files = || {
|
||||
let mut files = persisted_files(&aggregate.join("session"));
|
||||
files.remove(Path::new("artifacts/paste/.aggregate.lock"));
|
||||
files
|
||||
};
|
||||
let original = session_files();
|
||||
let mut bundle = test_bundle();
|
||||
bundle.metadata.workspace_id = "workspace-saved-subjektiv".into();
|
||||
bundle.prompt_catalog = Some(worker::PromptCatalog::builtins_only().unwrap().projection().clone());
|
||||
request.config_bundle = Some(bundle.with_computed_digest());
|
||||
|
||||
// Substitute only the Host transport. A committed attribution
|
||||
// with an undecodable response models lost acknowledgement;
|
||||
// its durable Session-keyed receipt survives the retry. This
|
||||
// does not test the real Host's signature or Subject policy.
|
||||
let attribution_path = fixture_root.join("host-attribution.json");
|
||||
let stored = attribution_path.clone();
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let record_calls = calls.clone();
|
||||
let startup_calls = Arc::new(AtomicUsize::new(0));
|
||||
let startups = startup_calls.clone();
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
request.request.workspace_api.as_mut().unwrap().base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let app = axum::Router::new()
|
||||
.route("/api/w/workspace-saved-subjektiv/authorization/check", axum::routing::post(
|
||||
|axum::Json(input): axum::Json<server_api::authorization::OperationAuthorizationRequest>| async move {
|
||||
axum::Json(saved_subjektiv_authorization_fixture(input).unwrap())
|
||||
}))
|
||||
.route("/api/w/workspace-saved-subjektiv/subjektiv/sessions", axum::routing::post(
|
||||
move |axum::Json(input): axum::Json<server_api::SubjektivRecordSessionRequest>| {
|
||||
let stored = stored.clone();
|
||||
let calls = record_calls.clone();
|
||||
async move {
|
||||
assert_eq!(input.session_id, session_id.to_string());
|
||||
let ordinal = calls.fetch_add(1, Ordering::SeqCst);
|
||||
if !stored.exists() {
|
||||
assert!(input.create_if_missing);
|
||||
fs::write(&stored, serde_json::to_vec(&server_api::SubjektivRecordSessionResponse {
|
||||
session_id: input.session_id, subject_id: "saved-subject".into(),
|
||||
}).unwrap()).unwrap();
|
||||
}
|
||||
(axum::http::StatusCode::OK, [(axum::http::header::CONTENT_TYPE, "application/json")],
|
||||
if ordinal == 0 { "{}".to_string() } else { fs::read_to_string(stored).unwrap() })
|
||||
}
|
||||
}))
|
||||
.route("/api/w/workspace-saved-subjektiv/subjektiv/journal/startup", axum::routing::get(move || {
|
||||
let startups = startups.clone();
|
||||
async move {
|
||||
startups.fetch_add(1, Ordering::SeqCst);
|
||||
axum::Json(serde_json::json!({ "subject_id": "saved-subject", "journals": [], "omitted": false }))
|
||||
}
|
||||
}));
|
||||
let host = HostProbe { calls: Arc::new(Mutex::new(Vec::new())), task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })) };
|
||||
// Same real factory and same operation for both repairs. No
|
||||
// candidate/controller is constructed by the test itself.
|
||||
assert!(factory.recover_saved_restore_source(&request).await.is_err());
|
||||
assert!(attribution_path.exists(), "Host committed before response loss");
|
||||
assert_eq!(session_files(), original);
|
||||
let failed = metadata.read_by_name(&name).unwrap().unwrap();
|
||||
assert!(matches!(failed.subjektiv_session_attribution,
|
||||
Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id));
|
||||
assert_eq!(startup_calls.load(Ordering::SeqCst), 0);
|
||||
factory.cleanup_failed_restore(&request).await.unwrap();
|
||||
assert!(factory.recover_saved_restore_source(&request).await.unwrap());
|
||||
let repaired = metadata.read_by_name(&name).unwrap().unwrap();
|
||||
assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id);
|
||||
assert!(matches!(repaired.subjektiv_session_attribution,
|
||||
Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, .. }) if id == session_id));
|
||||
let segments = WorkerSessionStore::new(aggregate.join("session")).unwrap()
|
||||
.list_segments(session_id).unwrap();
|
||||
assert_eq!(segments.len(), 1, "retry must retain exactly one seed");
|
||||
if materialized {
|
||||
assert_eq!(session_files(), original);
|
||||
assert_eq!(startup_calls.load(Ordering::SeqCst), 0);
|
||||
} else {
|
||||
assert_eq!(startup_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
let after_repair = persisted_files(&aggregate);
|
||||
let calls_after_repair = calls.load(Ordering::SeqCst);
|
||||
assert!(!factory.recover_saved_restore_source(&request).await.unwrap());
|
||||
assert_eq!(persisted_files(&aggregate), after_repair);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), calls_after_repair);
|
||||
assert!(factory.failed_restore_resources.lock().unwrap().is_empty());
|
||||
drop(host);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert_eq!(model.call_count.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
// Integration coverage through EmbeddedRuntime -> real Profile factory -> real
|
||||
// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O
|
||||
// is not invoked; this is a deterministic ordinary suite, not product E2E.
|
||||
// Worker/bootstrap, local Git Workdir provider, and allocation table. External
|
||||
// model I/O is replaced for saved-input coverage; this is not product E2E.
|
||||
|
||||
use crate::restore_test_environment::require_restore_test_environment;
|
||||
|
||||
fn restore_failure_tempdir() -> tempfile::TempDir {
|
||||
require_restore_test_environment();
|
||||
// Explicitly stay inside this checkout, even when TMPDIR points elsewhere.
|
||||
// Keep fixture components short; no symlink/alias to evade socket limits.
|
||||
let target = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
@@ -102,11 +105,16 @@ struct ProfileFailureFixture {
|
||||
backend: Arc<WorkerRuntimeExecutionBackend>,
|
||||
worker: crate::catalog::WorkerDetail,
|
||||
provider_opens: Arc<AtomicUsize>,
|
||||
seed_handle: Option<WorkerHandle>,
|
||||
root: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl ProfileFailureFixture {
|
||||
fn new(timeout: Duration) -> Self {
|
||||
Self::with_saved_inputs(timeout, None)
|
||||
}
|
||||
|
||||
fn with_saved_inputs(timeout: Duration, model: Option<MockClient>) -> Self {
|
||||
let root = restore_failure_tempdir();
|
||||
let repo = root.path().join("g");
|
||||
fs::create_dir(&repo).unwrap();
|
||||
@@ -118,10 +126,14 @@ impl ProfileFailureFixture {
|
||||
git(&repo, &["commit", "-m", "init"]);
|
||||
let store = root.path().join("s");
|
||||
let provider_opens = Arc::new(AtomicUsize::new(0));
|
||||
let mut factory =
|
||||
ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store);
|
||||
let queue_inputs = model.is_some();
|
||||
if let Some(model) = model {
|
||||
factory = factory.with_spawn_model_client_factory(move || Box::new(model.clone()));
|
||||
}
|
||||
let backend = Arc::new(
|
||||
WorkerRuntimeExecutionBackend::new(
|
||||
ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store),
|
||||
)
|
||||
WorkerRuntimeExecutionBackend::new(factory)
|
||||
.unwrap()
|
||||
.with_spawn_restore_timeout(timeout)
|
||||
.with_working_directory_materializer(RestoreOpenCountingMaterializer {
|
||||
@@ -146,12 +158,41 @@ impl ProfileFailureFixture {
|
||||
working_directory: working_directory_request(&repo),
|
||||
}];
|
||||
let worker = runtime.create_worker(request).unwrap();
|
||||
let seed_handle = queue_inputs.then(|| {
|
||||
backend.workers.lock().unwrap()[&worker.worker_ref]
|
||||
.handle
|
||||
.clone()
|
||||
});
|
||||
if let Some(handle) = seed_handle.clone() {
|
||||
backend.run_on_adapter_runtime(async move {
|
||||
async fn event(events: &mut tokio::sync::broadcast::Receiver<Event>, predicate: impl Fn(&Event) -> bool) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let next = events.recv().await.unwrap();
|
||||
assert!(!matches!(next, Event::Error { .. } | Event::SubmissionRejected { .. } | Event::NotificationRejected { .. }), "{next:?}");
|
||||
if predicate(&next) { break; }
|
||||
}
|
||||
}).await.expect("saved-input fixture acknowledgement");
|
||||
}
|
||||
let mut events = handle.subscribe();
|
||||
handle.send(Method::submit_text("revocation-seed", "hold for saved input setup")).await.unwrap();
|
||||
event(&mut events, |next| matches!(next, Event::TextDelta { .. })).await;
|
||||
handle.send(Method::Pause { command: protocol::WorkerCommandEnvelope::new(handle.shared_state.snapshot().last_command_id + 1) }).await.unwrap();
|
||||
event(&mut events, |next| matches!(next, Event::WorkerState { snapshot } if snapshot.catalog_status() == protocol::WorkerStatus::Paused)).await;
|
||||
handle.send(Method::submit_text("revocation-saved-submit", "must stay queued")).await.unwrap();
|
||||
event(&mut events, |next| matches!(next, Event::SubmissionAccepted { submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. } if submission_request_id == "revocation-saved-submit")).await;
|
||||
handle.send(Method::Notify { notification_request_id: "revocation-saved-notice".into(), message: "must also stay queued".into() }).await.unwrap();
|
||||
event(&mut events, |next| matches!(next, Event::NotificationAccepted { notification_request_id } if notification_request_id == "revocation-saved-notice")).await;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||
// Count restore provider access, not the initial create/stop setup.
|
||||
provider_opens.store(0, Ordering::SeqCst);
|
||||
Self {
|
||||
root,
|
||||
provider_opens,
|
||||
seed_handle,
|
||||
runtime,
|
||||
backend,
|
||||
worker,
|
||||
@@ -241,6 +282,68 @@ impl ProfileFailureFixture {
|
||||
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
fn resume_same_operation(
|
||||
&self,
|
||||
request: runtime_api::WorkerRestoreRequest,
|
||||
operation_id: crate::execution::WorkerLifecycleOperationId,
|
||||
) {
|
||||
let result = self
|
||||
.runtime
|
||||
.restore_worker_operation(&self.worker.worker_ref, request.clone())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.state,
|
||||
server_api::WorkerRestoreState::Accepted,
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
self.backend.workers.lock().unwrap()[&self.worker.worker_ref].restore_operation_id,
|
||||
Some(operation_id)
|
||||
);
|
||||
let calls = self
|
||||
.backend
|
||||
.factory
|
||||
.restore_faults
|
||||
.executions
|
||||
.load(Ordering::SeqCst);
|
||||
let replay = self
|
||||
.runtime
|
||||
.restore_worker_operation(&self.worker.worker_ref, request)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(replay).unwrap(),
|
||||
serde_json::to_value(result).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
self.backend
|
||||
.factory
|
||||
.restore_faults
|
||||
.executions
|
||||
.load(Ordering::SeqCst),
|
||||
calls
|
||||
);
|
||||
self.runtime
|
||||
.stop_worker(&self.worker.worker_ref, None)
|
||||
.unwrap();
|
||||
assert!(
|
||||
self.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.get(&self.worker.worker_ref)
|
||||
.is_none()
|
||||
);
|
||||
assert!(self.backend.pending_restores.lock().unwrap().is_empty());
|
||||
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||
assert!(
|
||||
self.backend
|
||||
.factory
|
||||
.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
fn fresh_restore_and_old_replay(
|
||||
&self,
|
||||
request: runtime_api::WorkerRestoreRequest,
|
||||
@@ -319,6 +422,171 @@ impl ProfileFailureFixture {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() {
|
||||
let model = MockClient::sequential(vec![MockResponse::Hang(
|
||||
simple_text_events().into_iter().take(2).collect(),
|
||||
)]);
|
||||
let fixture =
|
||||
ProfileFailureFixture::with_saved_inputs(Duration::from_secs(2), Some(model.clone()));
|
||||
let body_calls = model.call_count.load(Ordering::SeqCst);
|
||||
assert_eq!(body_calls, 1);
|
||||
let aggregate = fixture
|
||||
.backend
|
||||
.factory
|
||||
.worker_aggregate_dir(&fixture.worker.worker_ref)
|
||||
.unwrap();
|
||||
let session_before = persisted_files(&aggregate.join("session"));
|
||||
let metadata = WorkerAggregateStore::open_read_only(
|
||||
&aggregate,
|
||||
ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref),
|
||||
)
|
||||
.unwrap();
|
||||
let saved = metadata
|
||||
.read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(
|
||||
&fixture.worker.worker_ref,
|
||||
))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let (entered_tx, entered) = mpsc::channel();
|
||||
*fixture
|
||||
.backend
|
||||
.factory
|
||||
.restore_faults
|
||||
.factory_gate
|
||||
.lock()
|
||||
.unwrap() = Some(release.clone());
|
||||
*fixture
|
||||
.backend
|
||||
.factory
|
||||
.restore_faults
|
||||
.factory_entered
|
||||
.lock()
|
||||
.unwrap() = Some(entered_tx);
|
||||
let request = fixture
|
||||
.runtime
|
||||
.test_restore_request(&fixture.worker.worker_ref);
|
||||
let first = fixture
|
||||
.runtime
|
||||
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||
.unwrap();
|
||||
entered.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||
assert_eq!(
|
||||
first.state,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
);
|
||||
let pending = fixture
|
||||
.backend
|
||||
.pending_restore(&fixture.worker.worker_ref)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!pending.request.logical_workdir_attachments.is_empty());
|
||||
let updated = fixture
|
||||
.runtime
|
||||
.replace_worker_workdir_attachments(&fixture.worker.worker_ref, Vec::new())
|
||||
.unwrap();
|
||||
assert!(updated.workdir_attachments.is_empty());
|
||||
let authority = || -> serde_json::Value {
|
||||
serde_json::from_slice(
|
||||
&fs::read(
|
||||
fixture
|
||||
.root
|
||||
.path()
|
||||
.join("s/workers")
|
||||
.join(fixture.worker.worker_ref.worker_id.to_string())
|
||||
.join("worker.json"),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
assert!(authority().get("logical_workdir_attachments").is_none());
|
||||
release.notify_one();
|
||||
let replay = fixture
|
||||
.runtime
|
||||
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
replay.state,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||
"current attachment revocation must fence the old candidate: {replay:?}"
|
||||
);
|
||||
assert!(
|
||||
replay
|
||||
.message
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("current Worker authority differs"),
|
||||
"{replay:?}"
|
||||
);
|
||||
let record = authority();
|
||||
let owner = &record["restore_guard"]["owners"][&request.request_id];
|
||||
assert_eq!(
|
||||
owner["operation_id"],
|
||||
serde_json::to_value(pending.request.operation_id).unwrap()
|
||||
);
|
||||
assert!(owner["receipt"].is_null());
|
||||
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||
assert!(Arc::ptr_eq(
|
||||
&pending,
|
||||
&fixture
|
||||
.backend
|
||||
.pending_restore(&fixture.worker.worker_ref)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
));
|
||||
assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0);
|
||||
assert!(!pending.activation_started.load(Ordering::SeqCst));
|
||||
assert_eq!(model.call_count.load(Ordering::SeqCst), body_calls);
|
||||
assert_eq!(persisted_files(&aggregate.join("session")), session_before);
|
||||
assert_eq!(
|
||||
metadata.read_by_name(&saved.worker_name).unwrap().unwrap(),
|
||||
saved
|
||||
);
|
||||
let old = fixture.seed_handle.clone().unwrap();
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
assert!(old.protocol_is_closed());
|
||||
let old_command = Method::Resume {
|
||||
command: protocol::WorkerCommandEnvelope::new(
|
||||
old.shared_state.snapshot().last_command_id + 1,
|
||||
),
|
||||
};
|
||||
assert!(
|
||||
old.send(old_command).await.is_err(),
|
||||
"old execution command must not enter the restore candidate"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(authority().get("logical_workdir_attachments").is_none());
|
||||
let input = fixture.runtime.send_input(
|
||||
&fixture.worker.worker_ref,
|
||||
WorkerInput::user("must-not-enter-revoked-candidate"),
|
||||
);
|
||||
assert!(
|
||||
input.is_err(),
|
||||
"pending candidate must not admit fresh input: {input:?}"
|
||||
);
|
||||
// Stop owns cleanup of the original operation, never a newly rebound alias.
|
||||
fixture
|
||||
.runtime
|
||||
.stop_worker(&fixture.worker.worker_ref, None)
|
||||
.unwrap();
|
||||
fixture.assert_closed_deferred_sessions(&pending);
|
||||
assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0);
|
||||
assert!(fixture.backend.pending_restores.lock().unwrap().is_empty());
|
||||
let old = fixture
|
||||
.runtime
|
||||
.restore_worker_operation(&fixture.worker.worker_ref, request)
|
||||
.unwrap();
|
||||
assert_ne!(old.state, server_api::WorkerRestoreState::Accepted);
|
||||
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
|
||||
@@ -327,8 +595,6 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
|
||||
for stage in [
|
||||
RuntimeRestoreStage::Reconstruction,
|
||||
RuntimeRestoreStage::Binding,
|
||||
RuntimeRestoreStage::Feature,
|
||||
RuntimeRestoreStage::Controller,
|
||||
] {
|
||||
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||
fixture.fail(stage);
|
||||
@@ -474,7 +740,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.state,
|
||||
server_api::WorkerRestoreState::RolledBack,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||
"{result:?}"
|
||||
);
|
||||
assert!(
|
||||
@@ -488,12 +754,11 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer
|
||||
.iter()
|
||||
.all(|sessions| sessions.aliases().is_empty())
|
||||
);
|
||||
fixture.assert_released();
|
||||
fixture.assert_closed_deferred_sessions(&pending);
|
||||
eprintln!(
|
||||
"restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack"
|
||||
"restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=ReconciliationRequired"
|
||||
);
|
||||
fixture.fresh_restore_and_old_replay(request, result);
|
||||
fixture.resume_same_operation(request, pending.request.operation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -585,6 +850,7 @@ async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reuse
|
||||
FailedRestoreResources {
|
||||
operation_id: current.operation_id,
|
||||
sessions: Vec::new(),
|
||||
candidate_workdir_cleanup: None,
|
||||
allocation: None,
|
||||
feature_tasks: None,
|
||||
},
|
||||
@@ -715,14 +981,14 @@ fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconcile
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.state,
|
||||
server_api::WorkerRestoreState::RolledBack,
|
||||
server_api::WorkerRestoreState::Accepted,
|
||||
"{result:?}"
|
||||
);
|
||||
fixture.assert_released();
|
||||
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||
eprintln!(
|
||||
"restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1"
|
||||
);
|
||||
fixture.fresh_restore_and_old_replay(request, result);
|
||||
fixture.resume_same_operation(request, pending.request.operation_id);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -745,7 +1011,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.state,
|
||||
server_api::WorkerRestoreState::RolledBack,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||
"{result:?}"
|
||||
);
|
||||
let diagnostic = serde_json::to_string(&result).unwrap();
|
||||
@@ -757,8 +1023,12 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
||||
// This case deliberately exercises the real Unix socket path-limit error;
|
||||
// positive restore cases retain the factory's normal InProcess transport.
|
||||
// Short fixture roots must not accidentally turn it into a successful bind.
|
||||
let operation_id =
|
||||
fixture.backend.completed_restore_failures.lock().unwrap()[&fixture.worker.worker_ref].0;
|
||||
let pending = fixture
|
||||
.backend
|
||||
.pending_restore(&fixture.worker.worker_ref)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let operation_id = pending.request.operation_id;
|
||||
let socket = fixture
|
||||
.backend
|
||||
.factory
|
||||
@@ -767,7 +1037,25 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
||||
.join("worker.sock");
|
||||
assert!(socket.as_os_str().as_encoded_bytes().len() >= 108);
|
||||
assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string()));
|
||||
fixture.assert_released();
|
||||
assert!(!socket.exists());
|
||||
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.factory
|
||||
.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.completed_restore_failures
|
||||
.lock()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
fixture
|
||||
.backend
|
||||
.factory
|
||||
@@ -777,7 +1065,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
||||
eprintln!(
|
||||
"restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released"
|
||||
);
|
||||
fixture.fresh_restore_and_old_replay(request, result);
|
||||
fixture.resume_same_operation(request, operation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1178,6 +1466,8 @@ fn profile_restore_missing_committed_system_prompt_rolls_back_and_replays_classi
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let mut manifest = manifest;
|
||||
manifest.worker.name = worker_name.clone();
|
||||
metadata.resolved_manifest_snapshot =
|
||||
Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
|
||||
metadata_store.write(&metadata).unwrap();
|
||||
|
||||
@@ -3676,8 +3676,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
||||
crate::restore_test_environment::require_restore_test_environment();
|
||||
let repo = create_clean_repo();
|
||||
let runtime_root = tempfile::tempdir().unwrap();
|
||||
// Both ends use a short reference into this TempDir-owned tree. Keep
|
||||
// real agent/broker sockets and the existing bind/cleanup assertions.
|
||||
let runtime_root = tempfile::tempdir_in(".").unwrap();
|
||||
let runtime_path = Path::new(runtime_root.path().file_name().unwrap());
|
||||
let key_root = tempfile::tempdir().unwrap();
|
||||
let key_path = key_root.path().join("id_ed25519");
|
||||
let fallback_key_path = key_root.path().join("id_ed25519_fallback");
|
||||
@@ -3689,7 +3693,7 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(status.success());
|
||||
}
|
||||
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||
let materializer = RuntimeGitMaterializer::new(runtime_path);
|
||||
let mut request = request(repo.path());
|
||||
request.materialization = Some(crate::catalog::RepositoryMaterializationContext {
|
||||
workspace_id: "workspace-1".to_string(),
|
||||
@@ -3731,7 +3735,7 @@ mod tests {
|
||||
kind: server_api::RepositorySourceKind::Ssh,
|
||||
uri: "ssh://git@example.test/repo.git".to_string(),
|
||||
};
|
||||
let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation)
|
||||
let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!command_access.root.join("identity").exists());
|
||||
@@ -3873,18 +3877,17 @@ mod tests {
|
||||
.unwrap();
|
||||
// Restore preflight must validate without consuming this one-shot SSH
|
||||
// delivery or creating agent/session resources needed by actual bind.
|
||||
let record_before =
|
||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap();
|
||||
let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap();
|
||||
materializer
|
||||
.preflight_bind_working_directory(&id, None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
|
||||
fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
|
||||
record_before
|
||||
);
|
||||
assert_eq!(materializer.list_working_directories().unwrap().len(), 1);
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
0
|
||||
@@ -3898,7 +3901,7 @@ mod tests {
|
||||
assert!(ssh_policy.contains("__repository-ssh"));
|
||||
assert!(!ssh_policy.contains("SSH_AUTH_SOCK"));
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
1
|
||||
@@ -3916,7 +3919,7 @@ mod tests {
|
||||
"consumed delivery expiry must not revoke a live Workdir session"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
1,
|
||||
@@ -3925,7 +3928,7 @@ mod tests {
|
||||
drop(binding);
|
||||
assert!(!ssh_command.exists());
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
0
|
||||
@@ -3971,7 +3974,7 @@ mod tests {
|
||||
],
|
||||
&read_only_command_policy,
|
||||
));
|
||||
let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR))
|
||||
let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR))
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.map(|entry| entry.path().join("b.sock"))
|
||||
@@ -4104,7 +4107,7 @@ mod tests {
|
||||
.code,
|
||||
"working_directory_repository_access_expired"
|
||||
);
|
||||
let restored = RuntimeGitMaterializer::new(runtime_root.path());
|
||||
let restored = RuntimeGitMaterializer::new(runtime_path);
|
||||
assert_eq!(
|
||||
restored.bind_working_directory(&id, None).unwrap_err().code,
|
||||
"working_directory_remote_repository_access_required"
|
||||
|
||||
@@ -9,6 +9,7 @@ use crate::PromptCatalogSource;
|
||||
use crate::controller::{
|
||||
ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle,
|
||||
};
|
||||
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
|
||||
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
||||
use manifest::WorkerManifest;
|
||||
|
||||
@@ -168,6 +169,38 @@ where
|
||||
Ok(PreparedWorker::new(worker, self.layout, self.transport))
|
||||
}
|
||||
|
||||
/// Reconstruct a saved materialized Worker without allocation, attribution
|
||||
/// repair, Feature installation, or controller exposure. The caller owns the
|
||||
/// later controller layout/transport and must activate before starting it.
|
||||
pub async fn prepare_restored_candidate(
|
||||
self,
|
||||
worker_name: &str,
|
||||
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
|
||||
let mut candidate =
|
||||
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
|
||||
worker_name,
|
||||
self.manifest,
|
||||
self.store,
|
||||
self.prompt_catalog,
|
||||
self.workspace_context,
|
||||
self.filesystem_authority,
|
||||
self.model_client,
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
if let Some(workdir_session) = self.workdir_session {
|
||||
candidate
|
||||
.worker_mut()
|
||||
.map_err(WorkerBootstrapError::Worker)?
|
||||
.bind_single_workdir_session(Some(workdir_session));
|
||||
}
|
||||
candidate
|
||||
.worker_mut()
|
||||
.map_err(WorkerBootstrapError::Worker)?
|
||||
.bind_subjektiv_host(self.subjektiv_host);
|
||||
Ok(candidate)
|
||||
}
|
||||
|
||||
pub async fn prepare_restored(
|
||||
self,
|
||||
worker_name: &str,
|
||||
|
||||
@@ -689,6 +689,35 @@ pub enum WorkerControllerTransport {
|
||||
pub struct WorkerController;
|
||||
|
||||
impl WorkerController {
|
||||
/// Explicit recovery of an unmaterialized Session head. This installs the
|
||||
/// saved Feature composition needed to produce the seed, but creates no
|
||||
/// Controller, input channel, socket, observer, or queue consumer. The caller
|
||||
/// owns the same operation's Feature/session/allocation cleanup on every exit.
|
||||
pub async fn recover_pending_head<C, St>(
|
||||
worker: &mut Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
) -> std::io::Result<()>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
if worker.needs_initial_session_head_materialization() {
|
||||
worker
|
||||
.validate_subjektiv_body_publication()
|
||||
.map_err(std::io::Error::other)?;
|
||||
let registry = SpawnedWorkerRegistry::new_internal(
|
||||
worker.manifest().worker.name.clone(),
|
||||
worker.scope().clone(),
|
||||
);
|
||||
register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?;
|
||||
worker
|
||||
.materialize_durable_session_head()
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn spawn<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
|
||||
@@ -1066,7 +1066,8 @@ mod tests {
|
||||
use tokio::net::UnixListener;
|
||||
|
||||
use crate::runtime::dir::RuntimeDir;
|
||||
use crate::runtime::worker_allocation::test_util::RuntimeDirSandbox;
|
||||
|
||||
use crate::restore_test_environment::require_restore_test_environment;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FailTargetPeerStore {
|
||||
@@ -1110,11 +1111,13 @@ mod tests {
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn state_backed_visibility_and_restore_planning() {
|
||||
let root = TempDir::new().unwrap();
|
||||
require_restore_test_environment();
|
||||
let root = tempfile::tempdir_in(".").unwrap();
|
||||
let store_dir = root.path().join("store");
|
||||
let runtime_base = root.path().join("runtime");
|
||||
// Bind, durable child metadata, and discovery probes all use this same
|
||||
// short reference into the tree still owned by root. No cwd mutation.
|
||||
let runtime_base = Path::new(root.path().file_name().unwrap()).join("runtime");
|
||||
std::fs::create_dir_all(&runtime_base).unwrap();
|
||||
let _runtime_sandbox = RuntimeDirSandbox::new(&runtime_base);
|
||||
|
||||
let store = FsWorkerStore::new(&store_dir).unwrap();
|
||||
let session_id = new_session_id();
|
||||
|
||||
@@ -20,6 +20,9 @@ mod metadata_lifecycle;
|
||||
pub mod model_client;
|
||||
mod paste_artifact_tool;
|
||||
pub mod prompt;
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
pub mod runtime;
|
||||
pub mod runtime_command;
|
||||
pub mod segment_log_sink;
|
||||
|
||||
+1000
-26
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,4 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
use restore_test_environment::require_restore_test_environment;
|
||||
|
||||
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
|
||||
require_restore_test_environment();
|
||||
let temp = tempfile::tempdir_in(".").unwrap();
|
||||
// The relative reference and the TempDir name the same owned tree, without
|
||||
// putting the checkout prefix into sockaddr_un or changing process cwd.
|
||||
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||
(temp, runtime_base)
|
||||
}
|
||||
|
||||
fn simple_text_events() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
|
||||
manifest: WorkerManifest,
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
||||
require_restore_test_environment();
|
||||
let store_tmp = tempfile::tempdir().unwrap();
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(store_tmp.path()).unwrap(),
|
||||
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
|
||||
}
|
||||
|
||||
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let runtime_base = tmp.path().to_owned();
|
||||
std::mem::forget(tmp);
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
let (tmp, runtime_base) = owned_relative_runtime_base();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
.await
|
||||
.unwrap();
|
||||
// Retain ownership until controller shutdown (or test-runtime teardown).
|
||||
tokio::spawn(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
drop(tmp);
|
||||
});
|
||||
handle
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (root, runtime_base) = owned_relative_runtime_base();
|
||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
||||
let store = CombinedStore::new(
|
||||
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut rx = handle.subscribe();
|
||||
@@ -1761,11 +1779,12 @@ permission = "write"
|
||||
"#;
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let result = WorkerController::spawn(worker, tmp.path()).await;
|
||||
let (_tmp, runtime_base) = owned_relative_runtime_base();
|
||||
let result = WorkerController::spawn(worker, &runtime_base).await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"feature exposure must not imply delegation authority"
|
||||
"feature exposure must not imply delegation authority: {:?}",
|
||||
result.as_ref().err()
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,9 @@ pub mod repositories;
|
||||
pub mod repository_access;
|
||||
pub mod repository_source;
|
||||
pub mod resource_broker;
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
pub mod retention;
|
||||
pub mod runtime_settings;
|
||||
pub mod runtime_subscription;
|
||||
|
||||
@@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{
|
||||
WorkerRuntimeExecutionBackend,
|
||||
};
|
||||
|
||||
pub(super) fn connected_restore_tempdir() -> tempfile::TempDir {
|
||||
crate::restore_test_environment::require_restore_test_environment();
|
||||
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap();
|
||||
tempfile::Builder::new()
|
||||
.prefix("c")
|
||||
.rand_bytes(3)
|
||||
.tempdir_in(checkout.join("target"))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
const WATCHDOG: Duration = Duration::from_secs(45);
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -82,27 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory {
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.preflight_restore(request).await
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn recover_restore_source(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||
self.inner.recover_restore_source(request).await
|
||||
}
|
||||
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let worker_ref = request.worker_ref.clone();
|
||||
let inner = self.inner.prepare_restore(request).await?;
|
||||
Ok(Box::new(CapturingCandidate {
|
||||
inner,
|
||||
worker_ref,
|
||||
captures: self.captures.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cleanup_failed_restore(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.cleanup_failed_restore(request).await
|
||||
}
|
||||
|
||||
async fn reconcile_stopped_worker(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||
}
|
||||
|
||||
fn activate_restored_controller(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
workspace_id: Option<&str>,
|
||||
handle: &worker::WorkerHandle,
|
||||
) {
|
||||
self.inner
|
||||
.activate_restored_controller(worker_ref, workspace_id, handle);
|
||||
}
|
||||
}
|
||||
|
||||
struct CapturingCandidate {
|
||||
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_ref: worker_runtime::identity::WorkerRef,
|
||||
captures: Arc<Captures>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
self.inner.initial_state()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let worker_ref = request.worker_ref.clone();
|
||||
let controller = self.inner.restore_controller(request).await?;
|
||||
let controller = self.inner.activate().await?;
|
||||
self.captures
|
||||
.handles
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((worker_ref.clone(), controller.handle.clone()));
|
||||
.push((self.worker_ref.clone(), controller.handle.clone()));
|
||||
self.captures
|
||||
.restored_clients
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((worker_ref, controller.workspace_client.clone()));
|
||||
.push((self.worker_ref.clone(), controller.workspace_client.clone()));
|
||||
Ok(controller)
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.discard().await
|
||||
}
|
||||
}
|
||||
|
||||
struct Resources {
|
||||
@@ -262,7 +340,7 @@ async fn call(tools: &BTreeMap<String, Arc<dyn Tool>>, name: &str, args: Value)
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let root = connected_restore_tempdir();
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let mut config = test_server_config(root.path());
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -490,22 +490,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
|
||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||
.await
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
Ok(Box::new(AttentionRestoreCandidate {
|
||||
inputs: Some((
|
||||
AttentionWorkerFactory {
|
||||
root: self.root.clone(),
|
||||
},
|
||||
request.worker_ref.worker_id.to_string(),
|
||||
)),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// Preparation owns only fixture inputs; reconstruction and Controller startup
|
||||
// remain lazy so discard never needs to shut down a started Worker.
|
||||
struct AttentionRestoreCandidate {
|
||||
inputs: Option<(AttentionWorkerFactory, String)>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
protocol::WorkerStateSnapshot::initial()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||
"attention fixture candidate already consumed",
|
||||
)
|
||||
})?;
|
||||
factory.controller(worker_id, true).await.map_err(|_| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||
"attention fixture reconstruction failed",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inputs.take();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
use worker_runtime::execution as attention_execution;
|
||||
@@ -611,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() {
|
||||
|
||||
async fn real_attention_worker_start_restore(lose_response: bool) {
|
||||
use session_store::{LogEntry, Store, WorkerSessionStore};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dir = restored_parent_control_tests::connected_restore_tempdir();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let config = test_server_config(dir.path());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
|
||||
@@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() {
|
||||
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
||||
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
||||
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str());
|
||||
let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
||||
assert_eq!(page["items"].as_array().unwrap().len(),1);
|
||||
assert_eq!(page["items"][0]["model_processing_confirmed"],false);
|
||||
let path = format!(
|
||||
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",
|
||||
ticket.id.as_str()
|
||||
);
|
||||
let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||
assert_eq!(page["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(page["items"][0]["model_processing_confirmed"], false);
|
||||
assert!(page["items"][0].get("content").is_none());
|
||||
let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
||||
assert_eq!(again,page);
|
||||
let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap();
|
||||
assert_eq!(response.status(),StatusCode::UNAUTHORIZED);
|
||||
delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await;
|
||||
delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await;
|
||||
let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||
assert_eq!(again, page);
|
||||
let response = build_router(api.clone())
|
||||
.oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
delivery_http(
|
||||
&api,
|
||||
"GET",
|
||||
"/api/w/other-workspace/notification-diagnostics",
|
||||
None,
|
||||
StatusCode::NOT_FOUND,
|
||||
)
|
||||
.await;
|
||||
delivery_http(
|
||||
&api,
|
||||
"GET",
|
||||
&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),
|
||||
None,
|
||||
StatusCode::BAD_REQUEST,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn delivery_assignment_path(ticket: &str) -> String {
|
||||
@@ -828,22 +848,63 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
|
||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||
.await
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: delivery_execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
Ok(Box::new(DeliveryRestoreCandidate {
|
||||
inputs: Some((
|
||||
DeliveryWorkerFactory {
|
||||
root: self.root.clone(),
|
||||
client: self.client.clone(),
|
||||
},
|
||||
request.worker_ref.worker_id.to_string(),
|
||||
)),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// Retain the same scripted client/receipt store without opening either store or
|
||||
// starting a Controller until activation. Discard drops only these owned inputs.
|
||||
struct DeliveryRestoreCandidate {
|
||||
inputs: Option<(DeliveryWorkerFactory, String)>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
protocol::WorkerStateSnapshot::initial()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||
"delivery fixture candidate already consumed",
|
||||
)
|
||||
})?;
|
||||
factory.controller(worker_id, true).await.map_err(|_| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||
"delivery fixture reconstruction failed",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inputs.take();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Lose the response only AFTER the real Worker accepts the exact input. Unlike
|
||||
@@ -941,6 +1002,7 @@ async fn delivery_real_fixture(
|
||||
Arc<DeliveryResponseLossBackend>,
|
||||
DeliveryScriptedClient,
|
||||
) {
|
||||
let _environment = restored_parent_control_tests::connected_restore_tempdir();
|
||||
let config = test_server_config(root);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
@@ -1146,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
||||
"uncertain delivery changed request id or payload: {attempts:?}"
|
||||
);
|
||||
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
||||
let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery {
|
||||
assignment_id: Some(assignment.into()), ..Default::default()
|
||||
}).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt");
|
||||
let unknown = api
|
||||
.store
|
||||
.notification_diagnostics(
|
||||
TEST_WORKSPACE_ID,
|
||||
&server_api::NotificationDiagnosticsQuery {
|
||||
assignment_id: Some(assignment.into()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.items
|
||||
.into_iter()
|
||||
.find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown)
|
||||
.expect(
|
||||
"saved unknown acceptance must remain visible while Runtime already has its receipt",
|
||||
);
|
||||
assert!(!unknown.model_processing_confirmed);
|
||||
// Change mutable rendering context while the acknowledgement is unknown.
|
||||
// A new self-report is not a new input and cannot change the frozen retry.
|
||||
@@ -1209,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
||||
}
|
||||
delivery_wait_idle(&reopened, &worker, 2).await;
|
||||
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
||||
let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await;
|
||||
assert_eq!(evidence["items"][0]["stage"],"runtime_accepted");
|
||||
assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id));
|
||||
assert_eq!(evidence["items"][0]["model_processing_confirmed"],false);
|
||||
assert_eq!(evidence["items"][0]["retry_candidate"],false);
|
||||
let evidence = delivery_http(
|
||||
&reopened,
|
||||
"GET",
|
||||
&format!(
|
||||
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",
|
||||
unknown.source_id
|
||||
),
|
||||
None,
|
||||
StatusCode::OK,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(evidence["items"][0]["stage"], "runtime_accepted");
|
||||
assert_eq!(
|
||||
evidence["items"][0]["delivery_id"],
|
||||
json!(unknown.delivery_id)
|
||||
);
|
||||
assert_eq!(evidence["items"][0]["model_processing_confirmed"], false);
|
||||
assert_eq!(evidence["items"][0]["retry_candidate"], false);
|
||||
assert_eq!(
|
||||
delivery_commits(&reopened, &worker)
|
||||
.iter()
|
||||
|
||||
@@ -187,7 +187,14 @@ async fn config_call(
|
||||
runtime
|
||||
.invoke(
|
||||
path.into(),
|
||||
if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)},
|
||||
if matches!(operation, "edit" | "write") {
|
||||
wip_protocol::InterfaceReference {
|
||||
scope: "/".into(),
|
||||
name: format!("yoi.file/{operation}-text/v1"),
|
||||
}
|
||||
} else {
|
||||
config_interface(path)
|
||||
},
|
||||
operation.into(),
|
||||
args,
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
@@ -581,14 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
|
||||
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||
Ok(controller)
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn preflight_restore(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.preflight_restore(request).await
|
||||
}
|
||||
|
||||
async fn recover_restore_source(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||
self.inner.recover_restore_source(request).await
|
||||
}
|
||||
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let inner = self.inner.prepare_restore(request).await?;
|
||||
Ok(Box::new(ConfigCaptureCandidate {
|
||||
inner,
|
||||
client: self.client.clone(),
|
||||
controller_abort: self.controller_abort.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cleanup_failed_restore(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.cleanup_failed_restore(request).await
|
||||
}
|
||||
|
||||
async fn reconcile_stopped_worker(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||
}
|
||||
|
||||
fn activate_restored_controller(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
workspace_id: Option<&str>,
|
||||
handle: &worker::WorkerHandle,
|
||||
) {
|
||||
self.inner
|
||||
.activate_restored_controller(worker_ref, workspace_id, handle);
|
||||
}
|
||||
}
|
||||
|
||||
struct ConfigCaptureCandidate {
|
||||
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
client: Arc<Mutex<Option<Arc<dyn worker::WorkspaceClient>>>>,
|
||||
controller_abort: Arc<Mutex<Option<tokio::task::AbortHandle>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
self.inner.initial_state()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
self.inner.restore_controller(request).await
|
||||
let controller = self.inner.activate().await?;
|
||||
*self.client.lock().unwrap() = Some(controller.workspace_client.clone());
|
||||
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||
Ok(controller)
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.discard().await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -717,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
||||
decode_runtime_request_source_claims,
|
||||
};
|
||||
let root = restored_parent_control_tests::connected_restore_tempdir();
|
||||
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
||||
let mut fixture = manual_worker_assignment_fixture().await;
|
||||
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
||||
@@ -796,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
||||
|
||||
// Actual embedded Runtime factory -> real Worker controller -> captured
|
||||
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(root.path());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"tasks": {
|
||||
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts",
|
||||
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts",
|
||||
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
|
||||
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts"
|
||||
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts"
|
||||
},
|
||||
"fmt": {
|
||||
"proseWrap": "never"
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
//! Fail-closed preflight shared by restore fixtures across Worker, Runtime and Server.
|
||||
//!
|
||||
//! The normal runner supplies a separate namespace per process; a manually
|
||||
//! configured invocation may use the checkout-owned base itself. This module
|
||||
//! never sets process-wide environment or creates directories/tables.
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
fn validate_restore_test_paths(
|
||||
base: &Path,
|
||||
resolved: [Option<PathBuf>; 5],
|
||||
) -> Result<PathBuf, &'static str> {
|
||||
let root = resolved[0]
|
||||
.as_deref()
|
||||
.filter(|config| config.file_name().is_some_and(|name| name == "config"))
|
||||
.and_then(Path::parent)
|
||||
.ok_or("config")?;
|
||||
let namespace = root.strip_prefix(base).map_err(|_| "config")?;
|
||||
if namespace
|
||||
.components()
|
||||
.any(|part| !matches!(part, Component::Normal(_)))
|
||||
{
|
||||
return Err("config");
|
||||
}
|
||||
for ((label, suffix), actual) in [
|
||||
("config", "config"),
|
||||
("data", "data"),
|
||||
("secrets", "data"),
|
||||
("runtime", "run"),
|
||||
("allocation", "run/workers.json"),
|
||||
]
|
||||
.into_iter()
|
||||
.zip(resolved.iter())
|
||||
{
|
||||
if actual.as_deref() != Some(root.join(suffix).as_path()) {
|
||||
return Err(label);
|
||||
}
|
||||
}
|
||||
Ok(root.to_path_buf())
|
||||
}
|
||||
|
||||
fn require_owned_directory(checkout: &Path, directory: &Path) {
|
||||
let mut checked = checkout.to_path_buf();
|
||||
for part in directory.strip_prefix(checkout).unwrap().components() {
|
||||
assert!(matches!(part, Component::Normal(_)));
|
||||
checked.push(part);
|
||||
let metadata = fs::symlink_metadata(&checked)
|
||||
.expect("create isolated restore test directories before launching the test process");
|
||||
assert!(
|
||||
metadata.is_dir() && !metadata.file_type().is_symlink(),
|
||||
"restore test directory must not traverse symlinks"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn require_restore_test_environment() {
|
||||
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap();
|
||||
let base = checkout.join("target/restore-test-environment");
|
||||
// Compare all resolver results before inspecting even an absent allocation
|
||||
// table (its lookup can create a lock). Never inspect a live fallback path.
|
||||
let root = validate_restore_test_paths(
|
||||
&base,
|
||||
[
|
||||
manifest::paths::config_dir(),
|
||||
manifest::paths::data_dir(),
|
||||
manifest::paths::secret_data_dir(),
|
||||
manifest::paths::runtime_dir(),
|
||||
manifest::paths::worker_allocation_path(),
|
||||
],
|
||||
)
|
||||
.unwrap_or_else(|label| {
|
||||
panic!(
|
||||
"restore fixture requires isolated {label}: run deno task test, or configure YOI_CONFIG_DIR=<checkout>/target/restore-test-environment/config, YOI_DATA_DIR=<checkout>/target/restore-test-environment/data, YOI_RUNTIME_DIR=<checkout>/target/restore-test-environment/run before launch"
|
||||
)
|
||||
});
|
||||
assert_eq!(fs::canonicalize(checkout).unwrap(), checkout);
|
||||
for directory in [
|
||||
root.clone(),
|
||||
root.join("config"),
|
||||
root.join("data"),
|
||||
root.join("run"),
|
||||
] {
|
||||
require_owned_directory(checkout, &directory);
|
||||
}
|
||||
for name in ["workers.json", "workers.json.lock"] {
|
||||
match fs::symlink_metadata(root.join("run").join(name)) {
|
||||
Ok(metadata) => assert!(
|
||||
metadata.is_file() && !metadata.file_type().is_symlink(),
|
||||
"owned allocation paths must be regular files, not symlinks"
|
||||
),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => panic!("inspect owned restore allocation path: {error}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_fixture_path_validation_rejects_fallback_before_resource_access() {
|
||||
let base = Path::new("/owned/test-environment");
|
||||
for root in [base.to_path_buf(), base.join("run-1/worker-shard-2")] {
|
||||
let safe = ["config", "data", "data", "run", "run/workers.json"]
|
||||
.map(|suffix| Some(root.join(suffix)));
|
||||
assert_eq!(
|
||||
validate_restore_test_paths(base, safe.clone()),
|
||||
Ok(root.clone())
|
||||
);
|
||||
for index in 0..safe.len() {
|
||||
for fallback in [None, Some(PathBuf::from("/not-a-fixture"))] {
|
||||
let mut unsafe_paths = safe.clone();
|
||||
unsafe_paths[index] = fallback;
|
||||
assert!(validate_restore_test_paths(base, unsafe_paths).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
for root in [
|
||||
base.join("../escape"),
|
||||
base.join("safe/../../escape"),
|
||||
PathBuf::from("/outside"),
|
||||
] {
|
||||
let paths = ["config", "data", "data", "run", "run/workers.json"]
|
||||
.map(|suffix| Some(root.join(suffix)));
|
||||
assert!(validate_restore_test_paths(base, paths).is_err());
|
||||
}
|
||||
}
|
||||
+367
-2
@@ -6,6 +6,7 @@ import {
|
||||
mapConcurrent,
|
||||
parseJobs,
|
||||
parseRustListing,
|
||||
prepareTestIsolation,
|
||||
resultSummary,
|
||||
rustTestCommands,
|
||||
shardRustCommand,
|
||||
@@ -202,6 +203,39 @@ Deno.test("round-robin shards retain every listed name exactly once at threshold
|
||||
assertThrows(() => shardRustCommand(task, ["same", "same"]));
|
||||
});
|
||||
|
||||
Deno.test("global runtime-path packages serialize every normal target without losing names", () => {
|
||||
for (const pkg of ["worker", "worker-runtime", "yoi-workspace-server"]) {
|
||||
for (const target of ["lib", "controller_restore_test"]) {
|
||||
const task = {
|
||||
name: `${pkg}/${target}`,
|
||||
command: "/test-binary",
|
||||
args: [],
|
||||
cwd: "/pkg",
|
||||
};
|
||||
for (const count of [0, 1, 59, 72, 73, 1135]) {
|
||||
const names = listedNames(count);
|
||||
const shards = shardRustCommand(task, names);
|
||||
assert(
|
||||
shards.flatMap((shard) => shard.selectedNames).sort().join() ===
|
||||
[...names].sort().join(),
|
||||
);
|
||||
for (const shard of shards) {
|
||||
const args = shard.task.args;
|
||||
assert(args[args.indexOf("--test-threads") + 1] === "1");
|
||||
if (count > 72) {
|
||||
assert(args[0] === "--exact");
|
||||
assert(args.slice(3).join() === shard.selectedNames.join());
|
||||
} else {
|
||||
assert(args.join() === "--test-threads,1");
|
||||
}
|
||||
assert(!args.includes("--ignored") && !args.includes("--skip"));
|
||||
}
|
||||
}
|
||||
assert(task.args.length === 0, "do not mutate discovered commands");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("scheduler drains active workers before propagating a callback error", async () => {
|
||||
let drained = false;
|
||||
let failed = false;
|
||||
@@ -266,6 +300,276 @@ function emergencyKill(pid: number) {
|
||||
}
|
||||
}
|
||||
|
||||
async function assertRejects(
|
||||
run: () => Promise<unknown>,
|
||||
message: string,
|
||||
) {
|
||||
try {
|
||||
await run();
|
||||
} catch (error) {
|
||||
assert(String(error).includes(message), String(error));
|
||||
return;
|
||||
}
|
||||
throw new Error(`expected rejection containing ${message}`);
|
||||
}
|
||||
|
||||
Deno.test({
|
||||
name:
|
||||
"isolated real children use bounded disjoint tables and sanitized Git/compiler environments",
|
||||
ignore: Deno.build.os !== "linux",
|
||||
async fn() {
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
const environmentSnapshot = () =>
|
||||
JSON.stringify(
|
||||
Object.entries(process.env).sort(([a], [b]) => a.localeCompare(b)),
|
||||
);
|
||||
const before = environmentSnapshot();
|
||||
try {
|
||||
// An ancestor repository must not make an uninitialized tempfile look like
|
||||
// a repository. No developer repository or configuration is borrowed.
|
||||
for (const dir of [".git/objects", ".git/refs"]) {
|
||||
await Deno.mkdir(`${checkout}/${dir}`, { recursive: true });
|
||||
}
|
||||
await Deno.writeTextFile(
|
||||
`${checkout}/.git/HEAD`,
|
||||
"ref: refs/heads/main\n",
|
||||
);
|
||||
const [first, second] = await Promise.all([
|
||||
prepareTestIsolation(checkout, "same/output/id"),
|
||||
prepareTestIsolation(checkout, "same/output/id"),
|
||||
]);
|
||||
assert(
|
||||
first.root !== second.root,
|
||||
"concurrent runner ids must not collide",
|
||||
);
|
||||
const task = {
|
||||
...fixtureTask("isolation-env", checkout),
|
||||
env: {
|
||||
CARGO_MANIFEST_DIR: `${checkout}/crates/example`,
|
||||
CARGO_HOME: `${checkout}/compiler-cache`,
|
||||
RUSTUP_HOME: `${checkout}/toolchain-cache`,
|
||||
YOI_HOME: "/live/yoi",
|
||||
YOI_CONFIG_DIR: "/live/config",
|
||||
YOI_DATA_DIR: "/live/data",
|
||||
YOI_RUNTIME_DIR: "/live/run",
|
||||
HOME: "/live/home",
|
||||
TMPDIR: "/deleted/compiler-tmp",
|
||||
GIT_DIR: "/live/git",
|
||||
GIT_WORK_TREE: "/live/tree",
|
||||
GIT_CONFIG: "/live/gitconfig",
|
||||
GIT_CONFIG_GLOBAL: "/live/gitconfig",
|
||||
GIT_CONFIG_SYSTEM: "/live/system-gitconfig",
|
||||
GIT_TEMPLATE_DIR: "/live/template",
|
||||
GIT_CONFIG_PARAMETERS: "'core.hooksPath'='/live/hooks'",
|
||||
GIT_CONFIG_COUNT: "1",
|
||||
GIT_CONFIG_KEY_0: "core.hooksPath",
|
||||
GIT_CONFIG_VALUE_0: "/live/hooks",
|
||||
HTTP_PROXY: "http://live.invalid",
|
||||
https_proxy: "http://live.invalid",
|
||||
ALL_PROXY: "http://live.invalid",
|
||||
NO_PROXY: "live.invalid",
|
||||
RUSTC_WRAPPER: "/live/sccache",
|
||||
RUSTC_WORKSPACE_WRAPPER: "/live/wrapper",
|
||||
SCCACHE_DISABLE: "0",
|
||||
},
|
||||
};
|
||||
const taskBefore = JSON.stringify(task);
|
||||
const commands = await Promise.all([
|
||||
first.wrapCommand(task, "pkg/a"),
|
||||
first.wrapCommand(task, "pkg?a"), // same sanitized label
|
||||
first.wrapCommand(task, "pkg/a"), // same exact label
|
||||
second.wrapCommand(task, "pkg/a"),
|
||||
]);
|
||||
assert(JSON.stringify(task) === taskBefore);
|
||||
assert(
|
||||
environmentSnapshot() === before,
|
||||
"host environment changed",
|
||||
);
|
||||
const results = await Promise.all(
|
||||
commands.map((command, index) =>
|
||||
execute(command, {
|
||||
output: checkout,
|
||||
label: `environment-${index}`,
|
||||
timeoutMs: 10_000,
|
||||
})
|
||||
),
|
||||
);
|
||||
const tables = new Set<string>();
|
||||
for (const [index, result] of results.entries()) {
|
||||
assert(result.code === 0 && !result.error, JSON.stringify(result));
|
||||
const observed = JSON.parse(result.stdout);
|
||||
const env = observed.env as Record<string, string>;
|
||||
assert(observed.cwd === checkout && commands[index].cwd === task.cwd);
|
||||
assert(commands[index].args === task.args);
|
||||
assert(env.CARGO_MANIFEST_DIR === task.env.CARGO_MANIFEST_DIR);
|
||||
assert(env.CARGO_HOME === task.env.CARGO_HOME);
|
||||
assert(env.RUSTUP_HOME === task.env.RUSTUP_HOME);
|
||||
const namespace = env.YOI_RUNTIME_DIR.replace(/\/run$/, "");
|
||||
const prefix = `${checkout}/target/restore-test-environment/`;
|
||||
assert(namespace.startsWith(prefix));
|
||||
const suffix = namespace.slice(prefix.length);
|
||||
assert(
|
||||
suffix.split("/").every((part) => /^[a-zA-Z0-9_-]+$/.test(part)),
|
||||
);
|
||||
assert(env.YOI_CONFIG_DIR === `${namespace}/config`);
|
||||
assert(env.YOI_DATA_DIR === `${namespace}/data`);
|
||||
assert(
|
||||
env.YOI_HOME === env.YOI_DATA_DIR,
|
||||
"secret data must equal data",
|
||||
);
|
||||
assert(env.HOME === `${namespace}/home`);
|
||||
for (
|
||||
const dir of [
|
||||
namespace,
|
||||
env.YOI_CONFIG_DIR,
|
||||
env.YOI_DATA_DIR,
|
||||
env.YOI_RUNTIME_DIR,
|
||||
env.HOME,
|
||||
]
|
||||
) {
|
||||
assert((await Deno.lstat(dir)).isDirectory);
|
||||
assert(await Deno.realPath(dir) === dir);
|
||||
}
|
||||
for (const key of ["TMPDIR", "TMP", "TEMP"]) {
|
||||
assert(env[key] === `${checkout}/target/t`);
|
||||
}
|
||||
for (
|
||||
const key of [
|
||||
"GIT_DIR",
|
||||
"GIT_WORK_TREE",
|
||||
"GIT_CONFIG",
|
||||
"GIT_CONFIG_PARAMETERS",
|
||||
"GIT_CONFIG_COUNT",
|
||||
"GIT_CONFIG_KEY_0",
|
||||
"GIT_CONFIG_VALUE_0",
|
||||
"HTTP_PROXY",
|
||||
"https_proxy",
|
||||
"ALL_PROXY",
|
||||
"NO_PROXY",
|
||||
]
|
||||
) {
|
||||
assert(!(key in env), `${key} leaked to child`);
|
||||
}
|
||||
assert(env.GIT_CONFIG_GLOBAL === "/dev/null");
|
||||
assert(env.GIT_CONFIG_SYSTEM === "/dev/null");
|
||||
assert(env.GIT_CONFIG_NOSYSTEM === "1");
|
||||
assert(env.GIT_CEILING_DIRECTORIES === checkout);
|
||||
assert(env.GIT_TEMPLATE_DIR === `${namespace}/git-template`);
|
||||
assert(observed.globalConfig === "");
|
||||
assert(observed.ancestorDiscovered === false);
|
||||
assert(observed.hooks.length === 0 && observed.info.length === 0);
|
||||
assert(observed.author.includes("normal-tests@example.invalid"));
|
||||
assert(env.GIT_AUTHOR_NAME === env.GIT_COMMITTER_NAME);
|
||||
assert(env.GIT_AUTHOR_EMAIL === env.GIT_COMMITTER_EMAIL);
|
||||
assert(env.RUSTC_WRAPPER === "" && env.RUSTC_WORKSPACE_WRAPPER === "");
|
||||
assert(env.SCCACHE_DISABLE === "1");
|
||||
const table = `${env.YOI_RUNTIME_DIR}/workers.json`;
|
||||
tables.add(table);
|
||||
assert(await Deno.readTextFile(table) === String(observed.pid));
|
||||
}
|
||||
assert(
|
||||
tables.size === commands.length,
|
||||
"independent commands shared a table",
|
||||
);
|
||||
assert(environmentSnapshot() === before);
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
Deno.test("isolation refuses lexical aliases and symlinked owned paths before external writes", async () => {
|
||||
for (
|
||||
const unsafe of ["target", "target/restore-test-environment", "target/t"]
|
||||
) {
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
const outside = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
await Deno.writeTextFile(`${outside}/sentinel`, "untouched");
|
||||
if (unsafe !== "target") await Deno.mkdir(`${checkout}/target`);
|
||||
await Deno.symlink(outside, `${checkout}/${unsafe}`);
|
||||
await assertRejects(
|
||||
() => prepareTestIsolation(checkout, "run"),
|
||||
"symlink",
|
||||
);
|
||||
assert(
|
||||
Array.from(Deno.readDirSync(outside)).map((entry) => entry.name)
|
||||
.join() ===
|
||||
"sentinel",
|
||||
);
|
||||
assert(await Deno.readTextFile(`${outside}/sentinel`) === "untouched");
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
await Deno.remove(outside, { recursive: true });
|
||||
}
|
||||
}
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
await Deno.symlink(checkout, `${checkout}/alias`);
|
||||
await assertRejects(
|
||||
() => prepareTestIsolation(`${checkout}/alias`, "run"),
|
||||
"symlink",
|
||||
);
|
||||
for (const suffix of ["/../escape", "/./child", "//child"]) {
|
||||
await assertRejects(
|
||||
() => prepareTestIsolation(`${checkout}${suffix}`, "run"),
|
||||
"normal-only",
|
||||
);
|
||||
}
|
||||
assert(
|
||||
Array.from(Deno.readDirSync(checkout)).map((entry) => entry.name)
|
||||
.join() ===
|
||||
"alias",
|
||||
);
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("command wrapping rechecks owned ancestors and refuses symlinked namespace children", async () => {
|
||||
for (
|
||||
const unsafe of [
|
||||
"ancestor",
|
||||
"tmp",
|
||||
"config",
|
||||
"data",
|
||||
"run",
|
||||
"home",
|
||||
"git-template",
|
||||
"git-template/info",
|
||||
"git-template/hooks",
|
||||
]
|
||||
) {
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
const outside = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
const isolation = await prepareTestIsolation(checkout, "run");
|
||||
let path: string;
|
||||
if (unsafe === "ancestor") {
|
||||
path = isolation.root;
|
||||
await Deno.remove(path);
|
||||
} else if (unsafe === "tmp") {
|
||||
path = `${checkout}/target/t`;
|
||||
await Deno.remove(path);
|
||||
} else {
|
||||
const namespace = `${isolation.root}/unsafe-0`;
|
||||
await Deno.mkdir(namespace);
|
||||
if (unsafe.includes("/")) await Deno.mkdir(`${namespace}/git-template`);
|
||||
path = `${namespace}/${unsafe}`;
|
||||
}
|
||||
await Deno.symlink(outside, path);
|
||||
await assertRejects(
|
||||
() => isolation.wrapCommand(fixtureTask("success", checkout), "unsafe"),
|
||||
"symlink",
|
||||
);
|
||||
assert(Array.from(Deno.readDirSync(outside)).length === 0);
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
await Deno.remove(outside, { recursive: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) {
|
||||
Deno.test({
|
||||
name: `execute streams real ${mode} child output and records its exit`,
|
||||
@@ -436,15 +740,28 @@ Deno.test({
|
||||
"runtime listing persists complete name inventory for large and small binaries",
|
||||
ignore: Deno.build.os !== "linux",
|
||||
async fn() {
|
||||
const output = await Deno.makeTempDir();
|
||||
const output = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
const isolation = await prepareTestIsolation(output, "discovery");
|
||||
const wrappedTables = new Set<string>();
|
||||
const discovery = await discoverRustCommands(
|
||||
[
|
||||
fixtureTask("list-large", output),
|
||||
fixtureTask("list-small", output),
|
||||
],
|
||||
2,
|
||||
{ output },
|
||||
{
|
||||
output,
|
||||
run: async (task, label) => {
|
||||
const wrapped = await isolation.wrapCommand(task, label);
|
||||
wrappedTables.add(`${wrapped.env!.YOI_RUNTIME_DIR}/workers.json`);
|
||||
return await execute(wrapped, { output, label, timeoutMs: 10_000 });
|
||||
},
|
||||
},
|
||||
);
|
||||
assert(
|
||||
wrappedTables.size === 2,
|
||||
"all discovery launches must use the callback",
|
||||
);
|
||||
assert(discovery.inventory.complete);
|
||||
assert(
|
||||
@@ -635,6 +952,54 @@ if (import.meta.main) {
|
||||
);
|
||||
}
|
||||
Deno.exit(mode === "list-nonzero" ? 7 : 0);
|
||||
} else if (mode === "isolation-env") {
|
||||
const env = Deno.env.toObject();
|
||||
const scratch = await Deno.makeTempDir({ dir: env.TMPDIR });
|
||||
const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes);
|
||||
async function git(args: string[]) {
|
||||
return await new Deno.Command("git", {
|
||||
args,
|
||||
cwd: scratch,
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
}).output();
|
||||
}
|
||||
try {
|
||||
const ownedRepo = await git([
|
||||
"-C",
|
||||
output,
|
||||
"rev-parse",
|
||||
"--show-toplevel",
|
||||
]);
|
||||
assert(
|
||||
ownedRepo.success && text(ownedRepo.stdout).trim() === output,
|
||||
"owned ancestor repository fixture must be discoverable at its root",
|
||||
);
|
||||
const ancestor = await git(["rev-parse", "--show-toplevel"]);
|
||||
const init = await git(["init", "--quiet"]);
|
||||
assert(init.success, text(init.stderr));
|
||||
const global = await git(["config", "--global", "--list"]);
|
||||
assert(global.success, text(global.stderr));
|
||||
const author = await git(["var", "GIT_AUTHOR_IDENT"]);
|
||||
assert(author.success, text(author.stderr));
|
||||
await Deno.writeTextFile(
|
||||
`${env.YOI_RUNTIME_DIR}/workers.json`,
|
||||
String(Deno.pid),
|
||||
);
|
||||
console.log(JSON.stringify({
|
||||
env,
|
||||
cwd: Deno.cwd(),
|
||||
pid: Deno.pid,
|
||||
ancestorDiscovered: ancestor.success,
|
||||
globalConfig: text(global.stdout),
|
||||
author: text(author.stdout),
|
||||
hooks: Array.from(Deno.readDirSync(`${scratch}/.git/hooks`)),
|
||||
info: Array.from(Deno.readDirSync(`${scratch}/.git/info`)),
|
||||
}));
|
||||
} finally {
|
||||
await Deno.remove(scratch, { recursive: true });
|
||||
}
|
||||
Deno.exit(0);
|
||||
} else if (mode === "runner-parent") {
|
||||
const cancellation = installCancellationHandlers();
|
||||
try {
|
||||
|
||||
+196
-18
@@ -1,9 +1,10 @@
|
||||
// Bound top-level suites/shards, not total processes; large shards use two test threads.
|
||||
// Bound top-level suites/shards, not total processes; large shards normally use two threads.
|
||||
// Cargo selects targets/features. Retain all listed names; never skip cached successes.
|
||||
import { spawn } from "node:child_process";
|
||||
import process from "node:process";
|
||||
import { availableParallelism } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { posix } from "node:path";
|
||||
|
||||
// Safety limits, NOT the separate 60s performance goal.
|
||||
export const SUITE_TIMEOUT_MS = 180_000;
|
||||
@@ -65,7 +66,19 @@ export function shardRustCommand(
|
||||
if (new Set(names).size !== names.length) {
|
||||
throw new Error("cannot shard duplicate test names");
|
||||
}
|
||||
if (names.length <= 72) return [{ task, selectedNames: [...names] }];
|
||||
// RuntimeDirSandbox changes process-global paths. Do not overlap its writes
|
||||
// with restore guards in the same process, including small integration targets.
|
||||
const serial = ["worker", "worker-runtime", "yoi-workspace-server"].includes(
|
||||
task.name.split("/")[0],
|
||||
);
|
||||
if (names.length <= 72) {
|
||||
return [{
|
||||
task: serial
|
||||
? { ...task, args: [...task.args, "--test-threads", "1"] }
|
||||
: task,
|
||||
selectedNames: [...names],
|
||||
}];
|
||||
}
|
||||
const count = Math.min(32, Math.ceil(names.length / 36));
|
||||
const shards: string[][] = Array.from({ length: count }, () => []);
|
||||
names.forEach((name, index) => shards[index % count].push(name));
|
||||
@@ -75,7 +88,13 @@ export function shardRustCommand(
|
||||
name: `${task.name}/shard-${
|
||||
String(index + 1).padStart(2, "0")
|
||||
}-of-${count}`,
|
||||
args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames],
|
||||
args: [
|
||||
...task.args,
|
||||
"--exact",
|
||||
"--test-threads",
|
||||
serial ? "1" : "2",
|
||||
...selectedNames,
|
||||
],
|
||||
},
|
||||
selectedNames,
|
||||
}));
|
||||
@@ -84,18 +103,27 @@ export function shardRustCommand(
|
||||
export async function discoverRustCommands(
|
||||
commands: TestCommand[],
|
||||
jobs: number,
|
||||
options: { output: string; signal?: AbortSignal },
|
||||
options: {
|
||||
output: string;
|
||||
signal?: AbortSignal;
|
||||
run?: (task: TestCommand, label: string) => Promise<ExecutionResult>;
|
||||
},
|
||||
) {
|
||||
const binaries = await mapConcurrent(commands, jobs, async (task, index) => {
|
||||
const listing = await execute({
|
||||
const listingTask = {
|
||||
...task,
|
||||
name: `${task.name}/list`,
|
||||
args: [...task.args, "--list", "--format", "terse"],
|
||||
}, {
|
||||
...options,
|
||||
label: `list-${String(index).padStart(3, "0")}`,
|
||||
};
|
||||
const label = `list-${String(index).padStart(3, "0")}`;
|
||||
const listing = await (options.run
|
||||
? options.run(listingTask, label)
|
||||
: execute(listingTask, {
|
||||
output: options.output,
|
||||
signal: options.signal,
|
||||
label,
|
||||
timeoutMs: SUITE_TIMEOUT_MS,
|
||||
});
|
||||
}));
|
||||
let shards: RustShard[] = [];
|
||||
let error: string | undefined;
|
||||
try {
|
||||
@@ -124,7 +152,8 @@ export async function discoverRustCommands(
|
||||
error,
|
||||
selectedCases: error
|
||||
? null
|
||||
: shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0),
|
||||
: shards.reduce((sum, shard) =>
|
||||
sum + shard.selectedNames.length, 0),
|
||||
shards,
|
||||
};
|
||||
});
|
||||
@@ -426,12 +455,149 @@ export interface TestCommand {
|
||||
command: string;
|
||||
args: string[];
|
||||
cwd: string;
|
||||
env?: Record<string, string>;
|
||||
env?: Record<string, string | undefined>;
|
||||
// Exact link from each execution result into inventory.json, even when two
|
||||
// Cargo targets happen to share the same package/target display name.
|
||||
inventoryRef?: { binaryIndex: number; shardIndex: number };
|
||||
}
|
||||
|
||||
// Walk with lstat before creating any child: recursive mkdir/realPath alone
|
||||
// would follow an existing symlink first. Reject aliases rather than silently
|
||||
// canonicalizing a caller's checkout. As with the Rust guard, this assumes the
|
||||
// owned checkout is not concurrently replaced by an adversarial filesystem actor.
|
||||
async function ownedTestDirectory(checkout: string, relative: string) {
|
||||
if (
|
||||
!posix.isAbsolute(checkout) || checkout === "/" ||
|
||||
posix.normalize(checkout) !== checkout
|
||||
) throw new Error("test checkout must be an absolute normal-only path");
|
||||
|
||||
async function checkDirectory(path: string, owned: boolean) {
|
||||
const stat = await Deno.lstat(path);
|
||||
if (!stat.isDirectory || stat.isSymlink) {
|
||||
throw new Error(
|
||||
`test isolation refuses non-directory or symlink: ${path}`,
|
||||
);
|
||||
}
|
||||
if (owned && process.getuid && stat.uid !== process.getuid()) {
|
||||
throw new Error(
|
||||
`test isolation directory is not owned by this user: ${path}`,
|
||||
);
|
||||
}
|
||||
if (await Deno.realPath(path) !== path) {
|
||||
throw new Error(
|
||||
`test isolation refuses non-canonical directory: ${path}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
let checkoutPath = "";
|
||||
for (const part of checkout.slice(1).split("/")) {
|
||||
checkoutPath += `/${part}`;
|
||||
await checkDirectory(checkoutPath, false);
|
||||
}
|
||||
let path = checkout;
|
||||
for (const part of relative.split("/")) {
|
||||
if (!part || part === "." || part === "..") {
|
||||
throw new Error("invalid owned test directory component");
|
||||
}
|
||||
path += `/${part}`;
|
||||
try {
|
||||
await Deno.mkdir(path, { mode: 0o700 });
|
||||
} catch (error) {
|
||||
if (!(error instanceof Deno.errors.AlreadyExists)) throw error;
|
||||
}
|
||||
await checkDirectory(path, true);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Prepare child-only isolation for the maintained normal-test entrypoint.
|
||||
* Each wrap gets a fresh allocation table, even for repeated/sanitized labels or
|
||||
* concurrent runners using the same output id. Resources stay under target for
|
||||
* diagnosis; callers with temporary checkouts own their recursive cleanup.
|
||||
*/
|
||||
export async function prepareTestIsolation(checkout: string, runId: string) {
|
||||
const ownedDirectory = (relative: string) =>
|
||||
ownedTestDirectory(checkout, relative);
|
||||
const prefix = "target/restore-test-environment";
|
||||
await ownedDirectory(prefix);
|
||||
// Keep tempfile Unix-socket references short. tempfile owns unique resources
|
||||
// here; neither runners nor suites remove/replace this shared directory.
|
||||
const tmp = await ownedDirectory("target/t");
|
||||
const digest = new Uint8Array(
|
||||
await crypto.subtle.digest("SHA-256", new TextEncoder().encode(runId)),
|
||||
);
|
||||
const id = Array.from(
|
||||
digest.slice(0, 6),
|
||||
(byte) => byte.toString(16).padStart(2, "0"),
|
||||
).join("");
|
||||
const relativeRoot = `${prefix}/runner-${id}-${crypto.randomUUID()}`;
|
||||
const root = await ownedDirectory(relativeRoot);
|
||||
let next = 0;
|
||||
return {
|
||||
root,
|
||||
async wrapCommand(task: TestCommand, label: string): Promise<TestCommand> {
|
||||
const safeLabel = label.replaceAll(/[^a-zA-Z0-9_-]/g, "-").slice(0, 40) ||
|
||||
"command";
|
||||
const relative = `${relativeRoot}/${safeLabel}-${next++}`;
|
||||
// Recheck shared ancestors as well as the command namespace on every wrap.
|
||||
await ownedDirectory("target/t");
|
||||
const namespace = await ownedDirectory(relative);
|
||||
for (
|
||||
const dir of [
|
||||
"config",
|
||||
"data",
|
||||
"run",
|
||||
"home",
|
||||
"git-template/info",
|
||||
"git-template/hooks",
|
||||
]
|
||||
) {
|
||||
await ownedDirectory(`${relative}/${dir}`);
|
||||
}
|
||||
const inherited = { ...process.env, ...task.env };
|
||||
const env: NonNullable<TestCommand["env"]> = { ...task.env };
|
||||
// Undefined shadows inherited entries in execute's merge; node spawn omits
|
||||
// them. Strip indexed Git config and less common Git path overrides too.
|
||||
for (const key of Object.keys(inherited)) {
|
||||
if (/^GIT_/i.test(key) || /(^|_)proxy$/i.test(key)) {
|
||||
env[key] = undefined;
|
||||
}
|
||||
}
|
||||
Object.assign(env, {
|
||||
HOME: `${namespace}/home`,
|
||||
YOI_HOME: `${namespace}/data`,
|
||||
YOI_CONFIG_DIR: `${namespace}/config`,
|
||||
YOI_DATA_DIR: `${namespace}/data`,
|
||||
YOI_RUNTIME_DIR: `${namespace}/run`,
|
||||
TMPDIR: tmp,
|
||||
TMP: tmp,
|
||||
TEMP: tmp,
|
||||
GIT_TEMPLATE_DIR: `${namespace}/git-template`,
|
||||
GIT_CONFIG_GLOBAL: "/dev/null",
|
||||
GIT_CONFIG_SYSTEM: "/dev/null",
|
||||
GIT_CONFIG_NOSYSTEM: "1",
|
||||
GIT_ATTR_NOSYSTEM: "1",
|
||||
GIT_CEILING_DIRECTORIES: checkout,
|
||||
GIT_AUTHOR_NAME: "Normal test runner",
|
||||
GIT_AUTHOR_EMAIL: "normal-tests@example.invalid",
|
||||
GIT_COMMITTER_NAME: "Normal test runner",
|
||||
GIT_COMMITTER_EMAIL: "normal-tests@example.invalid",
|
||||
RUSTC_WRAPPER: "",
|
||||
RUSTC_WORKSPACE_WRAPPER: "",
|
||||
SCCACHE_DISABLE: "1",
|
||||
});
|
||||
// HOME is isolated, but toolchain/cache discovery must keep its ordinary
|
||||
// host locations (these are compiler settings, not live product state).
|
||||
for (const key of ["CARGO_HOME", "RUSTUP_HOME"] as const) {
|
||||
const fallback = key === "CARGO_HOME" ? ".cargo" : ".rustup";
|
||||
env[key] = inherited[key] ||
|
||||
(process.env.HOME ? `${process.env.HOME}/${fallback}` : undefined);
|
||||
}
|
||||
return { ...task, env };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function rustTestCommands(
|
||||
artifacts: CargoArtifact[],
|
||||
packages: { id: string; name: string; manifest_path: string }[],
|
||||
@@ -497,16 +663,26 @@ async function main() {
|
||||
"",
|
||||
);
|
||||
const jobs = parseJobs(Deno.args);
|
||||
const output = `${root}/target/normal-tests/${
|
||||
const outputId = `${
|
||||
new Date().toISOString().replaceAll(":", "-")
|
||||
}`;
|
||||
await Deno.mkdir(output, { recursive: true });
|
||||
}-${crypto.randomUUID()}`;
|
||||
const isolation = await prepareTestIsolation(root, outputId);
|
||||
const output = await ownedTestDirectory(
|
||||
root,
|
||||
`target/normal-tests/${outputId}`,
|
||||
);
|
||||
const cancellation = installCancellationHandlers();
|
||||
const run = (
|
||||
const run = async (
|
||||
task: TestCommand,
|
||||
label: string,
|
||||
timeoutMs = SUITE_TIMEOUT_MS,
|
||||
) => execute(task, { output, label, timeoutMs, signal: cancellation.signal });
|
||||
) =>
|
||||
execute(await isolation.wrapCommand(task, label), {
|
||||
output,
|
||||
label,
|
||||
timeoutMs,
|
||||
signal: cancellation.signal,
|
||||
});
|
||||
const summary: Record<string, unknown> = { phase: "metadata", failed: false };
|
||||
let executionStart: number | undefined;
|
||||
const writeSummary = () =>
|
||||
@@ -516,7 +692,7 @@ async function main() {
|
||||
{
|
||||
jobs,
|
||||
concurrency:
|
||||
"top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism",
|
||||
"top-level suites/shards; worker, worker-runtime and yoi-workspace-server use 1 test thread; other large Rust shards use 2, other suites retain internal parallelism",
|
||||
platform: "linux",
|
||||
suiteTimeoutMs: SUITE_TIMEOUT_MS,
|
||||
buildTimeoutMs: BUILD_TIMEOUT_MS,
|
||||
@@ -600,6 +776,7 @@ async function main() {
|
||||
const discovery = await discoverRustCommands(rust, jobs, {
|
||||
output,
|
||||
signal: cancellation.signal,
|
||||
run,
|
||||
});
|
||||
summary.discoverySeconds = (performance.now() - discoveryStart) / 1000;
|
||||
summary.inventory = "inventory.json";
|
||||
@@ -655,6 +832,7 @@ async function main() {
|
||||
"--allow-write",
|
||||
"--allow-run",
|
||||
"--allow-env",
|
||||
"--allow-sys=uid",
|
||||
"tools/test.test.ts",
|
||||
"tools/check-state-contract-names.test.ts",
|
||||
],
|
||||
@@ -663,7 +841,7 @@ async function main() {
|
||||
...discovery.tasks,
|
||||
];
|
||||
console.log(
|
||||
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
|
||||
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (worker/worker-runtime/yoi-workspace-server: 1 test thread; other large Rust shards: 2; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
|
||||
);
|
||||
const results: ReturnType<typeof resultSummary>[] = [];
|
||||
summary.results = results;
|
||||
|
||||
Reference in New Issue
Block a user