feat(worker): separate restored candidates, repair, and activation
This commit is contained in:
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
|
|||||||
Ok(store)
|
Ok(store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open a current retained Session without creating or migrating anything.
|
||||||
|
/// The live constraint is checked here and again by every later publication;
|
||||||
|
/// attaching the control store itself does not acquire publication authority.
|
||||||
|
pub fn open_existing_with_retention(
|
||||||
|
root: impl Into<PathBuf>,
|
||||||
|
retention: crate::retention::SessionRetentionStore,
|
||||||
|
) -> Result<Self, StoreError> {
|
||||||
|
let mut store = Self::open_read_only(root)?;
|
||||||
|
if let Some(session) = *store.session_id.lock().unwrap() {
|
||||||
|
retention.ensure_live(session)?;
|
||||||
|
}
|
||||||
|
store.retention = Some(retention);
|
||||||
|
Ok(store)
|
||||||
|
}
|
||||||
|
|
||||||
fn lock_publication(
|
fn lock_publication(
|
||||||
&self,
|
&self,
|
||||||
session: SessionId,
|
session: SessionId,
|
||||||
|
|||||||
@@ -2301,9 +2301,28 @@ impl WorkerAggregateRecord {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
if let Some(request) = request {
|
if let Some(request) = request {
|
||||||
|
let mut validation_request = request.clone();
|
||||||
|
// Pending admission may supply a missing Workspace connection
|
||||||
|
// without applying it to the saved execution. Validate that
|
||||||
|
// exact owner's intent against identity; do not widen the
|
||||||
|
// Available-record rule or mutate the canonical request.
|
||||||
|
if validation_request.workspace_api.is_none()
|
||||||
|
&& let PersistedWorkerLifecycleOperation::Restore(restore) = &operation
|
||||||
|
&& let Some(api) = restore_guard
|
||||||
|
.pending_preparation(restore.operation_id)
|
||||||
|
.and_then(|preparation| preparation.workspace_api.as_ref())
|
||||||
|
{
|
||||||
|
validation_request.workspace_api =
|
||||||
|
Some(
|
||||||
|
serde_json::from_value(serde_json::to_value(api).map_err(
|
||||||
|
|error| runtime_store_corrupt(path, error.to_string()),
|
||||||
|
)?)
|
||||||
|
.map_err(|error| runtime_store_corrupt(path, error.to_string()))?,
|
||||||
|
);
|
||||||
|
}
|
||||||
WorkerExecutionRecord {
|
WorkerExecutionRecord {
|
||||||
schema_version: SCHEMA_VERSION,
|
schema_version: SCHEMA_VERSION,
|
||||||
request: request.clone(),
|
request: validation_request,
|
||||||
binding: binding.clone(),
|
binding: binding.clone(),
|
||||||
restore_intent,
|
restore_intent,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ pub mod management;
|
|||||||
pub mod observation;
|
pub mod observation;
|
||||||
pub mod profile_archive;
|
pub mod profile_archive;
|
||||||
pub mod resource;
|
pub mod resource;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
pub mod retention;
|
pub mod retention;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
|
|||||||
@@ -682,8 +682,10 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
// The same TempDir owns the relative bind/connect reference. No cwd change
|
||||||
let root = temp.path().join("runtime");
|
// or alias is needed, and the real live-socket migration refusal stays tested.
|
||||||
|
let temp = tempfile::tempdir_in(".").unwrap();
|
||||||
|
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
let (worker, _, _) = runtime_fixture(&root, 12);
|
||||||
let before = tree_bytes(&root);
|
let before = tree_bytes(&root);
|
||||||
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,173 @@
|
|||||||
|
//! Explicit, operation-owned repair of saved Session material. This is not
|
||||||
|
//! read-only candidate preparation: the canonical Session/attribution is the
|
||||||
|
//! receipt for replay after a lost response, and effects are reported as such.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl ProfileRuntimeWorkerFactory {
|
||||||
|
pub(super) async fn recover_saved_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &WorkerExecutionRestoreRequest,
|
||||||
|
) -> Result<bool, RuntimeRestoreFailure> {
|
||||||
|
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
|
||||||
|
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
|
||||||
|
let root = self
|
||||||
|
.worker_aggregate_dir(&request.worker_ref)
|
||||||
|
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
|
||||||
|
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata open failed"))?;
|
||||||
|
let metadata = metadata_store
|
||||||
|
.read_by_name(&name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata read failed"))?
|
||||||
|
.ok_or_else(|| cause("restore recovery metadata missing"))?;
|
||||||
|
let active = metadata
|
||||||
|
.active
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("restore recovery Session missing"))?;
|
||||||
|
let pending = active.segment_id.is_none();
|
||||||
|
let incomplete = request.request.subjektiv_attached
|
||||||
|
&& !matches!(
|
||||||
|
metadata.subjektiv_session_attribution,
|
||||||
|
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
|
||||||
|
);
|
||||||
|
if !pending && !incomplete {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
// The accepted Runtime operation already owns this Session. Its stable
|
||||||
|
// Session identity and sole committed seed, not a new restore request,
|
||||||
|
// are used to reconcile an interrupted first-head publication.
|
||||||
|
let session_store = WorkerSessionStore::new_with_retention(
|
||||||
|
root.join("session"),
|
||||||
|
self.session_retention()
|
||||||
|
.map_err(|_| cause("restore recovery retention unavailable"))?,
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("restore recovery Session open failed"))?;
|
||||||
|
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
|
||||||
|
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
|
||||||
|
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
|
||||||
|
&request.request,
|
||||||
|
self.runtime_id.as_deref(),
|
||||||
|
);
|
||||||
|
let request_client = request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
|
||||||
|
let context = workspace.worker_context(
|
||||||
|
&request.worker_ref,
|
||||||
|
request.workspace_scope.as_ref(),
|
||||||
|
self.worker_mutation_identity.as_ref(),
|
||||||
|
request_client,
|
||||||
|
self.embedded_worker_mutation_dispatcher.as_ref(),
|
||||||
|
Some(self.prompt_projection_cache.clone()),
|
||||||
|
);
|
||||||
|
if pending && context.workspace_id().is_some() {
|
||||||
|
let bundle = request
|
||||||
|
.config_bundle
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("pending recovery launch material missing"))?;
|
||||||
|
let projection = self
|
||||||
|
.observe_bundle_prompt_projection(
|
||||||
|
bundle,
|
||||||
|
request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.map(|api| api.workspace_id.as_str()),
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
|
||||||
|
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
|
||||||
|
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
|
||||||
|
}
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: Vec::new(),
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: None,
|
||||||
|
feature_tasks: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let store = CombinedStore::new(session_store, metadata_store);
|
||||||
|
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
|
||||||
|
&name,
|
||||||
|
manifest,
|
||||||
|
store,
|
||||||
|
loader,
|
||||||
|
context,
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut worker = worker_result.map_err(|error| {
|
||||||
|
self.retain_construction_failure(request, &error);
|
||||||
|
cause(restore_worker_cause(&error))
|
||||||
|
})?;
|
||||||
|
let tasks = worker.retain_startup_feature_cleanup();
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
let allocation = worker.restore_allocation_cleanup_handle();
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: vec![sessions.clone()],
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: allocation.clone(),
|
||||||
|
feature_tasks: Some(tasks.clone()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let result = async {
|
||||||
|
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||||
|
worker
|
||||||
|
.bind_backend_job(binding.clone())
|
||||||
|
.map_err(|_| cause("recovery Backend Job binding failed"))?;
|
||||||
|
}
|
||||||
|
worker
|
||||||
|
.finalize_subjektiv_session_attribution(
|
||||||
|
request.request.subjektiv_attached,
|
||||||
|
SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||||
|
)
|
||||||
|
.map_err(|error| cause(restore_worker_cause(&error)))?;
|
||||||
|
if pending {
|
||||||
|
worker::WorkerController::recover_pending_head(&mut worker, &root)
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("pending Session head recovery failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
// Never release allocation while the Worker guard is attached. Retained
|
||||||
|
// handles in the same operation remain available on every cleanup error.
|
||||||
|
drop(worker);
|
||||||
|
let cleanup = async {
|
||||||
|
tasks
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Feature settlement failed"))?;
|
||||||
|
sessions
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Workdir close failed"))?;
|
||||||
|
if let Some(allocation) = allocation {
|
||||||
|
allocation
|
||||||
|
.release()
|
||||||
|
.map_err(|_| cause("recovery allocation release failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
result?;
|
||||||
|
cleanup?;
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.remove(&request.worker_ref);
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,898 @@
|
|||||||
|
// Included inside worker_backend::tests, alongside the restore failure suite.
|
||||||
|
// These integration tests connect the real Profile factory, canonical stores,
|
||||||
|
// restore owner, and Controller. Only model I/O and the Workspace HTTP transport
|
||||||
|
// are substituted. The loopback Host is not an authentication/policy test.
|
||||||
|
// Run in the ordinary worker-runtime lib suite with the default product features
|
||||||
|
// and the filter `restore_candidate::`. No live service is used. Like the
|
||||||
|
// failure suite, this requires the existing Yoi path settings to resolve into
|
||||||
|
// target/restore-test-environment; the common fixture rejects unsafe fallbacks
|
||||||
|
// before any factory, Git, or allocation-table operation.
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
mod restore_candidate {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
const FIRST: &str = "candidate-saved-first";
|
||||||
|
const SECOND: &str = "candidate-saved-second";
|
||||||
|
const NOTICE: &str = "candidate-saved-notice";
|
||||||
|
const FRESH: &str = "candidate-fresh-input";
|
||||||
|
const WAIT: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
|
struct HostProbe {
|
||||||
|
calls: Arc<Mutex<Vec<(String, String)>>>,
|
||||||
|
task: Option<tokio::task::JoinHandle<()>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for HostProbe {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Some(task) = self.task.take() {
|
||||||
|
task.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Aborting on unwind prevents a failed setup assertion from leaving its
|
||||||
|
// hanging model turn alive. Successful setup uses the normal shutdown/join.
|
||||||
|
struct SeedController(Option<RuntimeWorkerController>);
|
||||||
|
|
||||||
|
impl Drop for SeedController {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Some(controller) = self.0.as_ref() {
|
||||||
|
controller.controller_task.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn receive_matching(
|
||||||
|
events: &mut tokio::sync::broadcast::Receiver<Event>,
|
||||||
|
phase: &str,
|
||||||
|
mut matches: impl FnMut(&Event) -> bool,
|
||||||
|
) -> Event {
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
loop {
|
||||||
|
let event = events.recv().await.expect("Controller event stream");
|
||||||
|
seen.push(format!("{event:?}"));
|
||||||
|
if matches(&event) {
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!matches!(
|
||||||
|
event,
|
||||||
|
Event::Error { .. }
|
||||||
|
| Event::SubmissionRejected { .. }
|
||||||
|
| Event::NotificationRejected { .. }
|
||||||
|
),
|
||||||
|
"unexpected Controller rejection: {event:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| {
|
||||||
|
panic!("Controller acknowledgement deadline during {phase}; observed {seen:?}")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn submit(handle: &WorkerHandle, id: &str) -> Event {
|
||||||
|
let mut events = handle.subscribe();
|
||||||
|
handle
|
||||||
|
.send(Method::Submit {
|
||||||
|
submission_request_id: id.into(),
|
||||||
|
input: vec![protocol::Segment::Text { content: id.into() }],
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
receive_matching(&mut events, id, |event| {
|
||||||
|
matches!(event, Event::SubmissionAccepted { submission_request_id, .. }
|
||||||
|
if submission_request_id == id)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn notify(handle: &WorkerHandle) {
|
||||||
|
let mut events = handle.subscribe();
|
||||||
|
handle
|
||||||
|
.send(Method::Notify {
|
||||||
|
notification_request_id: NOTICE.into(),
|
||||||
|
message: NOTICE.into(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
receive_matching(&mut events, "notification acceptance", |event| {
|
||||||
|
matches!(event, Event::NotificationAccepted { notification_request_id }
|
||||||
|
if notification_request_id == NOTICE)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn pending(handle: &WorkerHandle) -> protocol::PendingSubmissionsSnapshot {
|
||||||
|
let mut events = handle.subscribe();
|
||||||
|
handle.send(Method::ListPendingSubmissions).await.unwrap();
|
||||||
|
match receive_matching(&mut events, "pending queue snapshot", |event| {
|
||||||
|
matches!(event, Event::PendingSubmissionsChanged { .. })
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Event::PendingSubmissionsChanged { pending } => pending,
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn finished(handle: &WorkerHandle, id: &str) {
|
||||||
|
let mut events = handle.subscribe();
|
||||||
|
let is_finished = |snapshot: &protocol::WorkerStateSnapshot| {
|
||||||
|
snapshot.last_finished_submission_request_id.as_deref() == Some(id)
|
||||||
|
&& snapshot.state == protocol::WorkerState::Idle
|
||||||
|
};
|
||||||
|
if !is_finished(&handle.shared_state.snapshot()) {
|
||||||
|
receive_matching(
|
||||||
|
&mut events,
|
||||||
|
id,
|
||||||
|
|event| matches!(event, Event::WorkerState { snapshot } if is_finished(snapshot)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn queue_checkpoint(entries: &[LogEntry]) -> serde_json::Value {
|
||||||
|
entries
|
||||||
|
.iter()
|
||||||
|
.rev()
|
||||||
|
.find_map(|entry| match entry {
|
||||||
|
LogEntry::Extension {
|
||||||
|
domain, payload, ..
|
||||||
|
} if domain == "worker.pending_activations.v1" => Some(payload.clone()),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.expect("real Controller must have committed its pending activation checkpoint")
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Fixture {
|
||||||
|
backend: Arc<WorkerRuntimeExecutionBackend>,
|
||||||
|
request: WorkerExecutionRestoreRequest,
|
||||||
|
model: MockClient,
|
||||||
|
session: WorkerSessionStore,
|
||||||
|
session_id: session_store::SessionId,
|
||||||
|
segment_id: session_store::SegmentId,
|
||||||
|
receipts: Vec<serde_json::Value>,
|
||||||
|
host: HostProbe,
|
||||||
|
_root: tempfile::TempDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Fixture {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self::with_transport(WorkerControllerTransport::InProcess)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_transport(transport: WorkerControllerTransport) -> Self {
|
||||||
|
let root = restore_failure_tempdir();
|
||||||
|
let model = MockClient::sequential(vec![
|
||||||
|
MockResponse::Complete(vec![
|
||||||
|
LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"),
|
||||||
|
LlmEvent::tool_input_delta(
|
||||||
|
0,
|
||||||
|
r#"{"subject":"Activated candidate","description":"Execution probe"}"#,
|
||||||
|
),
|
||||||
|
LlmEvent::tool_use_stop(0),
|
||||||
|
LlmEvent::Status(StatusEvent {
|
||||||
|
status: ResponseStatus::Completed,
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
MockResponse::Complete(simple_text_events()),
|
||||||
|
MockResponse::Complete(simple_text_events()),
|
||||||
|
MockResponse::Complete(simple_text_events()),
|
||||||
|
]);
|
||||||
|
let injected = model.clone();
|
||||||
|
let mut factory = ProfileRuntimeWorkerFactory::new(root.path())
|
||||||
|
.with_runtime_store_dir(root.path())
|
||||||
|
.with_runtime_id("candidate-runtime")
|
||||||
|
.with_spawn_model_client_factory(move || Box::new(injected.clone()));
|
||||||
|
factory.controller_transport = transport;
|
||||||
|
let catalog = worker::PromptCatalog::builtins_only()
|
||||||
|
.unwrap()
|
||||||
|
.projection()
|
||||||
|
.clone();
|
||||||
|
factory
|
||||||
|
.observe_workspace_prompt_projection(
|
||||||
|
worker::WorkspacePromptProjection::new(
|
||||||
|
"workspace-saved-subjektiv",
|
||||||
|
"candidate-fixture",
|
||||||
|
catalog.catalog_digest.clone(),
|
||||||
|
catalog,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
|
||||||
|
let fixture_root = root.path().to_path_buf();
|
||||||
|
let (request, session, session_id, segment_id, receipts, host) = backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
let authorization_calls = calls.clone();
|
||||||
|
let other_calls = calls.clone();
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
let authorization_path =
|
||||||
|
"/api/w/workspace-saved-subjektiv/authorization/check";
|
||||||
|
let app = axum::Router::new()
|
||||||
|
.route(
|
||||||
|
authorization_path,
|
||||||
|
axum::routing::post(move |axum::Json(input): axum::Json<
|
||||||
|
server_api::authorization::OperationAuthorizationRequest,
|
||||||
|
>| {
|
||||||
|
let calls = authorization_calls.clone();
|
||||||
|
async move {
|
||||||
|
calls.lock().unwrap().push((
|
||||||
|
"POST".into(), authorization_path.into(),
|
||||||
|
));
|
||||||
|
let response = if input.operation == "task"
|
||||||
|
&& input.resource == server_api::authorization::PermissionResource::Feature
|
||||||
|
{
|
||||||
|
server_api::authorization::OperationAuthorizationResponse {
|
||||||
|
scope: Some(server_api::authorization::PermissionResource::Feature),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
saved_subjektiv_authorization_fixture(input)
|
||||||
|
.expect("only the saved fixture Feature grants are available")
|
||||||
|
};
|
||||||
|
axum::Json(response)
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.fallback(move |request: axum::extract::Request| {
|
||||||
|
let calls = other_calls.clone();
|
||||||
|
async move {
|
||||||
|
calls.lock().unwrap().push((
|
||||||
|
request.method().to_string(), request.uri().path().to_owned(),
|
||||||
|
));
|
||||||
|
axum::http::StatusCode::FORBIDDEN
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let host = HostProbe {
|
||||||
|
calls,
|
||||||
|
task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })),
|
||||||
|
};
|
||||||
|
let worker_ref = WorkerRef::new(WorkerId::now_v7());
|
||||||
|
let (mut manifest, mut create) =
|
||||||
|
saved_subjektiv_policy(&fixture_root, &worker_ref);
|
||||||
|
// Normal restore deliberately excludes Subject repair. Keep
|
||||||
|
// a real Workspace client so accidental Host effects are visible.
|
||||||
|
manifest.feature.subjektiv = manifest::ResolvedSubjektivFeatureConfig::default();
|
||||||
|
manifest.feature.task.enabled = true;
|
||||||
|
create.subjektiv_attached = false;
|
||||||
|
create.workspace_api.as_mut().unwrap().base_url = base_url;
|
||||||
|
let aggregate = fixture_root.join("workers").join(worker_ref.worker_id.to_string());
|
||||||
|
let session = WorkerSessionStore::new(aggregate.join("session")).unwrap();
|
||||||
|
let metadata = WorkerAggregateStore::new(&aggregate, &manifest.worker.name).unwrap();
|
||||||
|
let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||||
|
MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]),
|
||||||
|
);
|
||||||
|
engine.set_system_prompt("Committed restore candidate fixture prompt");
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest.clone(),
|
||||||
|
engine,
|
||||||
|
CombinedStore::new(session.clone(), metadata.clone()),
|
||||||
|
RuntimeWorkspaceBackendRef::from_worker_request(&create, Some("candidate-runtime"))
|
||||||
|
.worker_context(&worker_ref, Some(&crate::runtime::RuntimeWorkspaceScope::new(
|
||||||
|
"workspace-saved-subjektiv", "server-main",
|
||||||
|
)), None, None, None, None),
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
Scope::empty(),
|
||||||
|
).await.unwrap();
|
||||||
|
worker.enable_worker_metadata_write_through().unwrap();
|
||||||
|
worker.materialize_durable_session_head().await.unwrap();
|
||||||
|
let (session_id, segment_id) = (worker.session_id(), worker.segment_id());
|
||||||
|
let workspace_client = worker.workspace_client_handle();
|
||||||
|
let started = PreparedWorker::new(
|
||||||
|
worker,
|
||||||
|
WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") },
|
||||||
|
WorkerControllerTransport::InProcess,
|
||||||
|
).start().await.unwrap();
|
||||||
|
let mut seed = SeedController(Some(RuntimeWorkerController {
|
||||||
|
handle: started.handle,
|
||||||
|
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||||
|
controller_task: started.controller_task,
|
||||||
|
workspace_client,
|
||||||
|
}));
|
||||||
|
let handle = seed.0.as_ref().unwrap().handle.clone();
|
||||||
|
let mut started_events = handle.subscribe();
|
||||||
|
submit(&handle, "candidate-setup-turn").await;
|
||||||
|
// Running and streaming observations can arrive in either
|
||||||
|
// order. Do not discard TextDelta while awaiting WorkerState.
|
||||||
|
let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running;
|
||||||
|
let mut streaming = false;
|
||||||
|
receive_matching(&mut started_events, "seed running and streaming", |event| {
|
||||||
|
if let Event::WorkerState { snapshot } = event {
|
||||||
|
running |= snapshot.catalog_status() == protocol::WorkerStatus::Running;
|
||||||
|
}
|
||||||
|
streaming |= matches!(event, Event::TextDelta { .. });
|
||||||
|
running && streaming
|
||||||
|
}).await;
|
||||||
|
let state = Arc::new(RwLock::new(handle.shared_state.snapshot()));
|
||||||
|
let mut events = handle.subscribe();
|
||||||
|
handle.send(Method::Pause { command: next_internal_command(&state).unwrap() })
|
||||||
|
.await.unwrap();
|
||||||
|
receive_matching(&mut events, "seed pause acknowledgement", |event| {
|
||||||
|
matches!(event, Event::WorkerState { snapshot }
|
||||||
|
if snapshot.catalog_status() == protocol::WorkerStatus::Paused)
|
||||||
|
}).await;
|
||||||
|
let mut receipts = Vec::new();
|
||||||
|
for id in [FIRST, SECOND] {
|
||||||
|
let receipt = submit(&handle, id).await;
|
||||||
|
assert!(matches!(receipt, Event::SubmissionAccepted {
|
||||||
|
disposition: protocol::SubmissionDisposition::Queued, ..
|
||||||
|
}));
|
||||||
|
receipts.push(serde_json::to_value(receipt).unwrap());
|
||||||
|
if id == FIRST { notify(&handle).await; }
|
||||||
|
}
|
||||||
|
assert_eq!(pending(&handle).await.submissions.len(), 2);
|
||||||
|
shutdown_profile_controller(seed.0.take().unwrap()).await;
|
||||||
|
// This orphan pin would be removed by the normal activation
|
||||||
|
// reconciliation. Preparation and discard must leave it alone.
|
||||||
|
let orphan = session.write_uploaded_file(
|
||||||
|
session_id, "orphan.txt", "text/plain", b"retained orphan",
|
||||||
|
session_store::UploadedFileLimits::default(),
|
||||||
|
).unwrap();
|
||||||
|
session.pin_uploaded_file(session_id, &orphan, "uncommitted-acceptance").unwrap();
|
||||||
|
let mut saved = metadata.read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref))
|
||||||
|
.unwrap().unwrap();
|
||||||
|
saved.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
|
||||||
|
saved.workspace_root = Some(fixture_root.join("stale-derived-root"));
|
||||||
|
metadata.write(&saved).unwrap();
|
||||||
|
let mut request = direct_restore_request(worker_ref);
|
||||||
|
request.request = create;
|
||||||
|
request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new(
|
||||||
|
"workspace-saved-subjektiv", "server-main",
|
||||||
|
));
|
||||||
|
Ok((request, session, session_id, segment_id, receipts, host))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let fixture = Self {
|
||||||
|
backend,
|
||||||
|
request,
|
||||||
|
model,
|
||||||
|
session,
|
||||||
|
session_id,
|
||||||
|
segment_id,
|
||||||
|
receipts,
|
||||||
|
host,
|
||||||
|
_root: root,
|
||||||
|
};
|
||||||
|
let checkpoint = queue_checkpoint(&fixture.entries());
|
||||||
|
assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
checkpoint["pending_notifications"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
fixture.assert_quiet();
|
||||||
|
fixture
|
||||||
|
}
|
||||||
|
|
||||||
|
fn entries(&self) -> Vec<LogEntry> {
|
||||||
|
self.session
|
||||||
|
.read_all(self.session_id, self.segment_id)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn canonical_files(&self) -> BTreeMap<PathBuf, Vec<u8>> {
|
||||||
|
let aggregate = self
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&self.request.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let mut files = persisted_files(&aggregate);
|
||||||
|
files.retain(|path, _| !path.components().any(|part| part.as_os_str() == "runs"));
|
||||||
|
files
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_no_modifying_host_calls(&self) {
|
||||||
|
assert!(
|
||||||
|
self.host
|
||||||
|
.calls
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.all(|(method, path)| {
|
||||||
|
method == "GET"
|
||||||
|
|| method == "HEAD"
|
||||||
|
|| (method == "POST"
|
||||||
|
&& path == "/api/w/workspace-saved-subjektiv/authorization/check")
|
||||||
|
}),
|
||||||
|
"unexpected modifying Host call"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_quiet(&self) {
|
||||||
|
assert_eq!(self.model.call_count.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(self.model.captured.lock().unwrap().is_empty());
|
||||||
|
self.assert_no_modifying_host_calls();
|
||||||
|
assert!(
|
||||||
|
session_store::collect_state(&self.entries())
|
||||||
|
.annotated_history
|
||||||
|
.iter()
|
||||||
|
.all(|entry| {
|
||||||
|
!matches!(
|
||||||
|
entry.item,
|
||||||
|
LoggedItem::ToolCall { .. } | LoggedItem::ToolResult { .. }
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
"prepare committed tool execution"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prepare(&self) {
|
||||||
|
assert!(matches!(
|
||||||
|
self.backend.restore_worker(self.request.clone()),
|
||||||
|
WorkerExecutionSpawnResult::Connected { .. }
|
||||||
|
));
|
||||||
|
// Connected here is the backend's prepared snapshot, not publication.
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.pending_restore(&self.request.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&self.request.worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!self
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_restore_run_dir(&self.request.worker_ref, self.request.operation_id,)
|
||||||
|
.unwrap()
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
let table = worker::runtime::worker_allocation::LockFileGuard::open(
|
||||||
|
&worker::runtime::worker_allocation::default_allocation_path().unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
table
|
||||||
|
.data()
|
||||||
|
.find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(
|
||||||
|
&self.request.worker_ref,
|
||||||
|
))
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_no_allocation(&self) {
|
||||||
|
let table = worker::runtime::worker_allocation::LockFileGuard::open(
|
||||||
|
&worker::runtime::worker_allocation::default_allocation_path().unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
table
|
||||||
|
.data()
|
||||||
|
.find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(
|
||||||
|
&self.request.worker_ref,
|
||||||
|
))
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_released(&self) {
|
||||||
|
self.assert_no_allocation();
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.pending_restore(&self.request.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&self.request.worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Fixture {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
// Keep the storage and loopback transport alive until owned tasks stop.
|
||||||
|
let _ = self.backend.stop_worker(&self.request.worker_ref);
|
||||||
|
if let Some(task) = self.host.task.take() {
|
||||||
|
task.abort();
|
||||||
|
let _ = self.backend.run_on_adapter_runtime(async move {
|
||||||
|
let _ = task.await;
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn prepare_saved_inputs_rejects_new_input_and_discard_preserves_authority() {
|
||||||
|
// Exercise the configured socket transport without ever publishing or
|
||||||
|
// binding it; activation coverage below uses the normal InProcess path.
|
||||||
|
let mut fixture = Fixture::with_transport(WorkerControllerTransport::UnixSocket);
|
||||||
|
let opens = Arc::new(AtomicUsize::new(0));
|
||||||
|
let counted = opens.clone();
|
||||||
|
let raw = deferred_workdir::DeferredWorkdirSession::handle(
|
||||||
|
"candidate-discard-resource",
|
||||||
|
workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
counted.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err("not opened during prepare".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
fixture.request.deferred_workdir_sessions.insert(
|
||||||
|
WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
|
raw.clone(),
|
||||||
|
);
|
||||||
|
let before = fixture.canonical_files();
|
||||||
|
fixture.prepare();
|
||||||
|
for input in [
|
||||||
|
WorkerInput::user("must-not-accept"),
|
||||||
|
WorkerInput::notify("must-not-notify"),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!fixture
|
||||||
|
.backend
|
||||||
|
.dispatch_input(&fixture.request.worker_ref, input)
|
||||||
|
.is_accepted()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
#[cfg(feature = "ws-server")]
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.clone()
|
||||||
|
.attach_worker_protocol(&fixture.request.worker_ref)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(!matches!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.worker_session(WorkerSessionObservationRequest {
|
||||||
|
worker_ref: fixture.request.worker_ref.clone(),
|
||||||
|
}),
|
||||||
|
runtime_api::WorkerSessionAvailability::LiveProtocol
|
||||||
|
));
|
||||||
|
fixture.assert_quiet();
|
||||||
|
assert_eq!(fixture.canonical_files(), before);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()[&fixture.request.worker_ref]
|
||||||
|
.candidate_workdir_cleanup
|
||||||
|
.is_some(),
|
||||||
|
"retain the real prepared Worker's close-only resource owner"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.stop_worker(&fixture.request.worker_ref)
|
||||||
|
.is_accepted()
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
|
||||||
|
let factory = fixture.backend.factory.clone();
|
||||||
|
let request = fixture.request.clone();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
// The adapter resolves and owns its own raw handles. Separately
|
||||||
|
// exercise the factory seam with the exact supplied cleanup handle.
|
||||||
|
let mut candidate = factory
|
||||||
|
.prepare_restore(request)
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
candidate
|
||||||
|
.discard()
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
assert!(matches!(
|
||||||
|
raw.read(workdir::ReadRequest {
|
||||||
|
path: workdir::WorkdirPath::new("probe.txt").unwrap(),
|
||||||
|
offset: 0,
|
||||||
|
limit: 1,
|
||||||
|
max_bytes: 32,
|
||||||
|
})
|
||||||
|
.await,
|
||||||
|
Err(workdir::WorkdirError::SessionClosed)
|
||||||
|
));
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(opens.load(Ordering::SeqCst), 0);
|
||||||
|
fixture.assert_quiet();
|
||||||
|
assert_eq!(fixture.canonical_files(), before);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.stop_worker(&fixture.request.worker_ref)
|
||||||
|
.is_accepted()
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.canonical_files(), before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn prepare_failure_at_each_pre_activation_checkpoint_preserves_saved_authority() {
|
||||||
|
// Feature and Controller are activation checkpoints, not preparation.
|
||||||
|
for stage in [
|
||||||
|
RuntimeRestoreStage::Metadata,
|
||||||
|
RuntimeRestoreStage::Reconstruction,
|
||||||
|
RuntimeRestoreStage::Binding,
|
||||||
|
] {
|
||||||
|
let fixture = Fixture::new();
|
||||||
|
let before = fixture.canonical_files();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.fail_at
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(stage);
|
||||||
|
let result = fixture.backend.restore_worker(fixture.request.clone());
|
||||||
|
assert!(
|
||||||
|
matches!(result, WorkerExecutionSpawnResult::RolledBack(_)),
|
||||||
|
"{stage:?}: {result:?}"
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture.assert_quiet();
|
||||||
|
assert_eq!(fixture.canonical_files(), before, "{stage:?}");
|
||||||
|
let completed = fixture.backend.completed_restore_failures.lock().unwrap();
|
||||||
|
let (_, failure) = &completed[&fixture.request.worker_ref];
|
||||||
|
assert_eq!(failure.stage, stage);
|
||||||
|
assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn activate_saved_inputs_drains_fifo_once_and_replays_durable_receipts() {
|
||||||
|
let fixture = Fixture::new();
|
||||||
|
let before = fixture.canonical_files();
|
||||||
|
fixture.prepare();
|
||||||
|
fixture.assert_quiet();
|
||||||
|
assert_eq!(fixture.canonical_files(), before);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.activate_restored_worker(fixture.request.operation_id, &fixture.request.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref]
|
||||||
|
.handle
|
||||||
|
.clone();
|
||||||
|
let model = fixture.model.clone();
|
||||||
|
let receipts = fixture.receipts.clone();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
finished(&handle, SECOND).await;
|
||||||
|
assert_eq!(model.call_count.load(Ordering::SeqCst), 3);
|
||||||
|
{
|
||||||
|
let requests = model.captured.lock().unwrap();
|
||||||
|
let first = serde_json::to_string(&requests[0].items).unwrap();
|
||||||
|
let second = serde_json::to_string(&requests[2].items).unwrap();
|
||||||
|
assert!(
|
||||||
|
requests[0]
|
||||||
|
.tools
|
||||||
|
.iter()
|
||||||
|
.any(|tool| tool.name == "TaskCreate")
|
||||||
|
);
|
||||||
|
assert!(first.contains(FIRST));
|
||||||
|
assert!(!first.contains(SECOND), "second Submit overtook the first");
|
||||||
|
assert_eq!(first.matches(NOTICE).count(), 1);
|
||||||
|
assert!(second.find(FIRST).unwrap() < second.find(SECOND).unwrap());
|
||||||
|
assert_eq!(second.matches(NOTICE).count(), 1);
|
||||||
|
}
|
||||||
|
let committed = serde_json::to_value(handle.committed_entries()).unwrap();
|
||||||
|
for (id, mut expected) in [FIRST, SECOND].into_iter().zip(receipts) {
|
||||||
|
// Activation advances the durable receipt from Queued to Started;
|
||||||
|
// replay must preserve its original submission identity.
|
||||||
|
expected["data"]["disposition"] = serde_json::json!("started");
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(submit(&handle, id).await).unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
notify(&handle).await;
|
||||||
|
assert!(pending(&handle).await.submissions.is_empty());
|
||||||
|
assert_eq!(model.call_count.load(Ordering::SeqCst), 3);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(handle.committed_entries()).unwrap(),
|
||||||
|
committed
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let history = session_store::collect_state(&fixture.entries()).annotated_history;
|
||||||
|
assert!(
|
||||||
|
history.iter().any(|entry| matches!(
|
||||||
|
&entry.item,
|
||||||
|
LoggedItem::ToolResult { call_id, is_error: false, .. }
|
||||||
|
if call_id == "candidate-task-probe"
|
||||||
|
)),
|
||||||
|
"activation must execute the real TaskCreate tool, not just decode a call"
|
||||||
|
);
|
||||||
|
for marker in [FIRST, SECOND, NOTICE] {
|
||||||
|
let count = history
|
||||||
|
.iter()
|
||||||
|
.filter(|entry| {
|
||||||
|
match &entry.item {
|
||||||
|
LoggedItem::Message { content, .. } => content.iter().any(|part| {
|
||||||
|
matches!(part, LoggedContentPart::Text { text } if text.contains(marker))
|
||||||
|
}),
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.count();
|
||||||
|
assert_eq!(count, 1, "{marker} must commit exactly once");
|
||||||
|
}
|
||||||
|
let mut fresh = WorkerInput::user(FRESH);
|
||||||
|
fresh.submission_request_id = Some(FRESH.into());
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.dispatch_input(&fixture.request.worker_ref, fresh)
|
||||||
|
.is_accepted()
|
||||||
|
);
|
||||||
|
let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref]
|
||||||
|
.handle
|
||||||
|
.clone();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
finished(&handle, FRESH).await;
|
||||||
|
assert!(pending(&handle).await.submissions.is_empty());
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(fixture.model.call_count.load(Ordering::SeqCst), 4);
|
||||||
|
assert!(
|
||||||
|
!fixture.host.calls.lock().unwrap().is_empty(),
|
||||||
|
"activation must reach the configured Host transport"
|
||||||
|
);
|
||||||
|
fixture.assert_no_modifying_host_calls();
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.stop_worker(&fixture.request.worker_ref)
|
||||||
|
.is_accepted()
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn explicit_repair_reconciles_lost_attribution_response_without_reseeding() {
|
||||||
|
for materialized in [true, false] {
|
||||||
|
let root = restore_failure_tempdir();
|
||||||
|
let model = MockClient::new(Vec::new());
|
||||||
|
let injected = model.clone();
|
||||||
|
let backend = WorkerRuntimeExecutionBackend::new(
|
||||||
|
ProfileRuntimeWorkerFactory::new(root.path())
|
||||||
|
.with_runtime_store_dir(root.path())
|
||||||
|
.with_runtime_id("repair-runtime")
|
||||||
|
.with_spawn_model_client_factory(move || Box::new(injected.clone())),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let factory = backend.factory.clone();
|
||||||
|
let fixture_root = root.path().to_path_buf();
|
||||||
|
backend.run_on_adapter_runtime(async move {
|
||||||
|
let mut request = saved_subjektiv_restore_fixture(&fixture_root, materialized).await;
|
||||||
|
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref);
|
||||||
|
let aggregate = fixture_root.join("workers").join(request.worker_ref.worker_id.to_string());
|
||||||
|
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
|
||||||
|
let mut saved = metadata.read_by_name(&name).unwrap().unwrap();
|
||||||
|
let session_id = saved.active.as_ref().unwrap().session_id;
|
||||||
|
saved.subjektiv_session_attribution = Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id, diagnostic: "saved interrupted attribution".into() });
|
||||||
|
metadata.write(&saved).unwrap();
|
||||||
|
// Explicit repair may acquire artifact locks; compare canonical
|
||||||
|
// contents separately from that lock-file side effect.
|
||||||
|
let session_files = || {
|
||||||
|
let mut files = persisted_files(&aggregate.join("session"));
|
||||||
|
files.remove(Path::new("artifacts/paste/.aggregate.lock"));
|
||||||
|
files
|
||||||
|
};
|
||||||
|
let original = session_files();
|
||||||
|
let mut bundle = test_bundle();
|
||||||
|
bundle.metadata.workspace_id = "workspace-saved-subjektiv".into();
|
||||||
|
bundle.prompt_catalog = Some(worker::PromptCatalog::builtins_only().unwrap().projection().clone());
|
||||||
|
request.config_bundle = Some(bundle.with_computed_digest());
|
||||||
|
|
||||||
|
// Substitute only the Host transport. A committed attribution
|
||||||
|
// with an undecodable response models lost acknowledgement;
|
||||||
|
// its durable Session-keyed receipt survives the retry. This
|
||||||
|
// does not test the real Host's signature or Subject policy.
|
||||||
|
let attribution_path = fixture_root.join("host-attribution.json");
|
||||||
|
let stored = attribution_path.clone();
|
||||||
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let record_calls = calls.clone();
|
||||||
|
let startup_calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let startups = startup_calls.clone();
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
request.request.workspace_api.as_mut().unwrap().base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
let app = axum::Router::new()
|
||||||
|
.route("/api/w/workspace-saved-subjektiv/authorization/check", axum::routing::post(
|
||||||
|
|axum::Json(input): axum::Json<server_api::authorization::OperationAuthorizationRequest>| async move {
|
||||||
|
axum::Json(saved_subjektiv_authorization_fixture(input).unwrap())
|
||||||
|
}))
|
||||||
|
.route("/api/w/workspace-saved-subjektiv/subjektiv/sessions", axum::routing::post(
|
||||||
|
move |axum::Json(input): axum::Json<server_api::SubjektivRecordSessionRequest>| {
|
||||||
|
let stored = stored.clone();
|
||||||
|
let calls = record_calls.clone();
|
||||||
|
async move {
|
||||||
|
assert_eq!(input.session_id, session_id.to_string());
|
||||||
|
let ordinal = calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if !stored.exists() {
|
||||||
|
assert!(input.create_if_missing);
|
||||||
|
fs::write(&stored, serde_json::to_vec(&server_api::SubjektivRecordSessionResponse {
|
||||||
|
session_id: input.session_id, subject_id: "saved-subject".into(),
|
||||||
|
}).unwrap()).unwrap();
|
||||||
|
}
|
||||||
|
(axum::http::StatusCode::OK, [(axum::http::header::CONTENT_TYPE, "application/json")],
|
||||||
|
if ordinal == 0 { "{}".to_string() } else { fs::read_to_string(stored).unwrap() })
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.route("/api/w/workspace-saved-subjektiv/subjektiv/journal/startup", axum::routing::get(move || {
|
||||||
|
let startups = startups.clone();
|
||||||
|
async move {
|
||||||
|
startups.fetch_add(1, Ordering::SeqCst);
|
||||||
|
axum::Json(serde_json::json!({ "subject_id": "saved-subject", "journals": [], "omitted": false }))
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let host = HostProbe { calls: Arc::new(Mutex::new(Vec::new())), task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })) };
|
||||||
|
// Same real factory and same operation for both repairs. No
|
||||||
|
// candidate/controller is constructed by the test itself.
|
||||||
|
assert!(factory.recover_saved_restore_source(&request).await.is_err());
|
||||||
|
assert!(attribution_path.exists(), "Host committed before response loss");
|
||||||
|
assert_eq!(session_files(), original);
|
||||||
|
let failed = metadata.read_by_name(&name).unwrap().unwrap();
|
||||||
|
assert!(matches!(failed.subjektiv_session_attribution,
|
||||||
|
Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id));
|
||||||
|
assert_eq!(startup_calls.load(Ordering::SeqCst), 0);
|
||||||
|
factory.cleanup_failed_restore(&request).await.unwrap();
|
||||||
|
assert!(factory.recover_saved_restore_source(&request).await.unwrap());
|
||||||
|
let repaired = metadata.read_by_name(&name).unwrap().unwrap();
|
||||||
|
assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id);
|
||||||
|
assert!(matches!(repaired.subjektiv_session_attribution,
|
||||||
|
Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, .. }) if id == session_id));
|
||||||
|
let segments = WorkerSessionStore::new(aggregate.join("session")).unwrap()
|
||||||
|
.list_segments(session_id).unwrap();
|
||||||
|
assert_eq!(segments.len(), 1, "retry must retain exactly one seed");
|
||||||
|
if materialized {
|
||||||
|
assert_eq!(session_files(), original);
|
||||||
|
assert_eq!(startup_calls.load(Ordering::SeqCst), 0);
|
||||||
|
} else {
|
||||||
|
assert_eq!(startup_calls.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
let after_repair = persisted_files(&aggregate);
|
||||||
|
let calls_after_repair = calls.load(Ordering::SeqCst);
|
||||||
|
assert!(!factory.recover_saved_restore_source(&request).await.unwrap());
|
||||||
|
assert_eq!(persisted_files(&aggregate), after_repair);
|
||||||
|
assert_eq!(calls.load(Ordering::SeqCst), calls_after_repair);
|
||||||
|
assert!(factory.failed_restore_resources.lock().unwrap().is_empty());
|
||||||
|
drop(host);
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert_eq!(model.call_count.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,11 @@
|
|||||||
// Integration coverage through EmbeddedRuntime -> real Profile factory -> real
|
// Integration coverage through EmbeddedRuntime -> real Profile factory -> real
|
||||||
// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O
|
// Worker/bootstrap, local Git Workdir provider, and allocation table. External
|
||||||
// is not invoked; this is a deterministic ordinary suite, not product E2E.
|
// model I/O is replaced for saved-input coverage; this is not product E2E.
|
||||||
|
|
||||||
|
use crate::restore_test_environment::require_restore_test_environment;
|
||||||
|
|
||||||
fn restore_failure_tempdir() -> tempfile::TempDir {
|
fn restore_failure_tempdir() -> tempfile::TempDir {
|
||||||
|
require_restore_test_environment();
|
||||||
// Explicitly stay inside this checkout, even when TMPDIR points elsewhere.
|
// Explicitly stay inside this checkout, even when TMPDIR points elsewhere.
|
||||||
// Keep fixture components short; no symlink/alias to evade socket limits.
|
// Keep fixture components short; no symlink/alias to evade socket limits.
|
||||||
let target = Path::new(env!("CARGO_MANIFEST_DIR"))
|
let target = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
@@ -102,11 +105,16 @@ struct ProfileFailureFixture {
|
|||||||
backend: Arc<WorkerRuntimeExecutionBackend>,
|
backend: Arc<WorkerRuntimeExecutionBackend>,
|
||||||
worker: crate::catalog::WorkerDetail,
|
worker: crate::catalog::WorkerDetail,
|
||||||
provider_opens: Arc<AtomicUsize>,
|
provider_opens: Arc<AtomicUsize>,
|
||||||
|
seed_handle: Option<WorkerHandle>,
|
||||||
root: tempfile::TempDir,
|
root: tempfile::TempDir,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProfileFailureFixture {
|
impl ProfileFailureFixture {
|
||||||
fn new(timeout: Duration) -> Self {
|
fn new(timeout: Duration) -> Self {
|
||||||
|
Self::with_saved_inputs(timeout, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_saved_inputs(timeout: Duration, model: Option<MockClient>) -> Self {
|
||||||
let root = restore_failure_tempdir();
|
let root = restore_failure_tempdir();
|
||||||
let repo = root.path().join("g");
|
let repo = root.path().join("g");
|
||||||
fs::create_dir(&repo).unwrap();
|
fs::create_dir(&repo).unwrap();
|
||||||
@@ -118,16 +126,20 @@ impl ProfileFailureFixture {
|
|||||||
git(&repo, &["commit", "-m", "init"]);
|
git(&repo, &["commit", "-m", "init"]);
|
||||||
let store = root.path().join("s");
|
let store = root.path().join("s");
|
||||||
let provider_opens = Arc::new(AtomicUsize::new(0));
|
let provider_opens = Arc::new(AtomicUsize::new(0));
|
||||||
|
let mut factory =
|
||||||
|
ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store);
|
||||||
|
let queue_inputs = model.is_some();
|
||||||
|
if let Some(model) = model {
|
||||||
|
factory = factory.with_spawn_model_client_factory(move || Box::new(model.clone()));
|
||||||
|
}
|
||||||
let backend = Arc::new(
|
let backend = Arc::new(
|
||||||
WorkerRuntimeExecutionBackend::new(
|
WorkerRuntimeExecutionBackend::new(factory)
|
||||||
ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store),
|
.unwrap()
|
||||||
)
|
.with_spawn_restore_timeout(timeout)
|
||||||
.unwrap()
|
.with_working_directory_materializer(RestoreOpenCountingMaterializer {
|
||||||
.with_spawn_restore_timeout(timeout)
|
inner: RuntimeGitMaterializer::new(root.path().join("w")),
|
||||||
.with_working_directory_materializer(RestoreOpenCountingMaterializer {
|
opens: provider_opens.clone(),
|
||||||
inner: RuntimeGitMaterializer::new(root.path().join("w")),
|
}),
|
||||||
opens: provider_opens.clone(),
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||||
crate::fs_store::FsRuntimeStoreOptions {
|
crate::fs_store::FsRuntimeStoreOptions {
|
||||||
@@ -146,12 +158,41 @@ impl ProfileFailureFixture {
|
|||||||
working_directory: working_directory_request(&repo),
|
working_directory: working_directory_request(&repo),
|
||||||
}];
|
}];
|
||||||
let worker = runtime.create_worker(request).unwrap();
|
let worker = runtime.create_worker(request).unwrap();
|
||||||
|
let seed_handle = queue_inputs.then(|| {
|
||||||
|
backend.workers.lock().unwrap()[&worker.worker_ref]
|
||||||
|
.handle
|
||||||
|
.clone()
|
||||||
|
});
|
||||||
|
if let Some(handle) = seed_handle.clone() {
|
||||||
|
backend.run_on_adapter_runtime(async move {
|
||||||
|
async fn event(events: &mut tokio::sync::broadcast::Receiver<Event>, predicate: impl Fn(&Event) -> bool) {
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
loop {
|
||||||
|
let next = events.recv().await.unwrap();
|
||||||
|
assert!(!matches!(next, Event::Error { .. } | Event::SubmissionRejected { .. } | Event::NotificationRejected { .. }), "{next:?}");
|
||||||
|
if predicate(&next) { break; }
|
||||||
|
}
|
||||||
|
}).await.expect("saved-input fixture acknowledgement");
|
||||||
|
}
|
||||||
|
let mut events = handle.subscribe();
|
||||||
|
handle.send(Method::submit_text("revocation-seed", "hold for saved input setup")).await.unwrap();
|
||||||
|
event(&mut events, |next| matches!(next, Event::TextDelta { .. })).await;
|
||||||
|
handle.send(Method::Pause { command: protocol::WorkerCommandEnvelope::new(handle.shared_state.snapshot().last_command_id + 1) }).await.unwrap();
|
||||||
|
event(&mut events, |next| matches!(next, Event::WorkerState { snapshot } if snapshot.catalog_status() == protocol::WorkerStatus::Paused)).await;
|
||||||
|
handle.send(Method::submit_text("revocation-saved-submit", "must stay queued")).await.unwrap();
|
||||||
|
event(&mut events, |next| matches!(next, Event::SubmissionAccepted { submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. } if submission_request_id == "revocation-saved-submit")).await;
|
||||||
|
handle.send(Method::Notify { notification_request_id: "revocation-saved-notice".into(), message: "must also stay queued".into() }).await.unwrap();
|
||||||
|
event(&mut events, |next| matches!(next, Event::NotificationAccepted { notification_request_id } if notification_request_id == "revocation-saved-notice")).await;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
// Count restore provider access, not the initial create/stop setup.
|
// Count restore provider access, not the initial create/stop setup.
|
||||||
provider_opens.store(0, Ordering::SeqCst);
|
provider_opens.store(0, Ordering::SeqCst);
|
||||||
Self {
|
Self {
|
||||||
root,
|
root,
|
||||||
provider_opens,
|
provider_opens,
|
||||||
|
seed_handle,
|
||||||
runtime,
|
runtime,
|
||||||
backend,
|
backend,
|
||||||
worker,
|
worker,
|
||||||
@@ -241,6 +282,68 @@ impl ProfileFailureFixture {
|
|||||||
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
|
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resume_same_operation(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
|
operation_id: crate::execution::WorkerLifecycleOperationId,
|
||||||
|
) {
|
||||||
|
let result = self
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&self.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
self.backend.workers.lock().unwrap()[&self.worker.worker_ref].restore_operation_id,
|
||||||
|
Some(operation_id)
|
||||||
|
);
|
||||||
|
let calls = self
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst);
|
||||||
|
let replay = self
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&self.worker.worker_ref, request)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(replay).unwrap(),
|
||||||
|
serde_json::to_value(result).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
calls
|
||||||
|
);
|
||||||
|
self.runtime
|
||||||
|
.stop_worker(&self.worker.worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&self.worker.worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(self.backend.pending_restores.lock().unwrap().is_empty());
|
||||||
|
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn fresh_restore_and_old_replay(
|
fn fresh_restore_and_old_replay(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::WorkerRestoreRequest,
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
@@ -319,6 +422,171 @@ impl ProfileFailureFixture {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() {
|
||||||
|
let model = MockClient::sequential(vec![MockResponse::Hang(
|
||||||
|
simple_text_events().into_iter().take(2).collect(),
|
||||||
|
)]);
|
||||||
|
let fixture =
|
||||||
|
ProfileFailureFixture::with_saved_inputs(Duration::from_secs(2), Some(model.clone()));
|
||||||
|
let body_calls = model.call_count.load(Ordering::SeqCst);
|
||||||
|
assert_eq!(body_calls, 1);
|
||||||
|
let aggregate = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&fixture.worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let session_before = persisted_files(&aggregate.join("session"));
|
||||||
|
let metadata = WorkerAggregateStore::open_read_only(
|
||||||
|
&aggregate,
|
||||||
|
ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let saved = metadata
|
||||||
|
.read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(
|
||||||
|
&fixture.worker.worker_ref,
|
||||||
|
))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let (entered_tx, entered) = mpsc::channel();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(release.clone());
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_entered
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(entered_tx);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let first = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
entered.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(!pending.request.logical_workdir_attachments.is_empty());
|
||||||
|
let updated = fixture
|
||||||
|
.runtime
|
||||||
|
.replace_worker_workdir_attachments(&fixture.worker.worker_ref, Vec::new())
|
||||||
|
.unwrap();
|
||||||
|
assert!(updated.workdir_attachments.is_empty());
|
||||||
|
let authority = || -> serde_json::Value {
|
||||||
|
serde_json::from_slice(
|
||||||
|
&fs::read(
|
||||||
|
fixture
|
||||||
|
.root
|
||||||
|
.path()
|
||||||
|
.join("s/workers")
|
||||||
|
.join(fixture.worker.worker_ref.worker_id.to_string())
|
||||||
|
.join("worker.json"),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
assert!(authority().get("logical_workdir_attachments").is_none());
|
||||||
|
release.notify_one();
|
||||||
|
let replay = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
replay.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"current attachment revocation must fence the old candidate: {replay:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
replay
|
||||||
|
.message
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.contains("current Worker authority differs"),
|
||||||
|
"{replay:?}"
|
||||||
|
);
|
||||||
|
let record = authority();
|
||||||
|
let owner = &record["restore_guard"]["owners"][&request.request_id];
|
||||||
|
assert_eq!(
|
||||||
|
owner["operation_id"],
|
||||||
|
serde_json::to_value(pending.request.operation_id).unwrap()
|
||||||
|
);
|
||||||
|
assert!(owner["receipt"].is_null());
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&pending,
|
||||||
|
&fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
));
|
||||||
|
assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(!pending.activation_started.load(Ordering::SeqCst));
|
||||||
|
assert_eq!(model.call_count.load(Ordering::SeqCst), body_calls);
|
||||||
|
assert_eq!(persisted_files(&aggregate.join("session")), session_before);
|
||||||
|
assert_eq!(
|
||||||
|
metadata.read_by_name(&saved.worker_name).unwrap().unwrap(),
|
||||||
|
saved
|
||||||
|
);
|
||||||
|
let old = fixture.seed_handle.clone().unwrap();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
assert!(old.protocol_is_closed());
|
||||||
|
let old_command = Method::Resume {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(
|
||||||
|
old.shared_state.snapshot().last_command_id + 1,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
old.send(old_command).await.is_err(),
|
||||||
|
"old execution command must not enter the restore candidate"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(authority().get("logical_workdir_attachments").is_none());
|
||||||
|
let input = fixture.runtime.send_input(
|
||||||
|
&fixture.worker.worker_ref,
|
||||||
|
WorkerInput::user("must-not-enter-revoked-candidate"),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
input.is_err(),
|
||||||
|
"pending candidate must not admit fresh input: {input:?}"
|
||||||
|
);
|
||||||
|
// Stop owns cleanup of the original operation, never a newly rebound alias.
|
||||||
|
fixture
|
||||||
|
.runtime
|
||||||
|
.stop_worker(&fixture.worker.worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_closed_deferred_sessions(&pending);
|
||||||
|
assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(fixture.backend.pending_restores.lock().unwrap().is_empty());
|
||||||
|
let old = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request)
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(old.state, server_api::WorkerRestoreState::Accepted);
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial(worker_allocation)]
|
#[serial_test::serial(worker_allocation)]
|
||||||
fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
|
fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
|
||||||
@@ -327,8 +595,6 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
|
|||||||
for stage in [
|
for stage in [
|
||||||
RuntimeRestoreStage::Reconstruction,
|
RuntimeRestoreStage::Reconstruction,
|
||||||
RuntimeRestoreStage::Binding,
|
RuntimeRestoreStage::Binding,
|
||||||
RuntimeRestoreStage::Feature,
|
|
||||||
RuntimeRestoreStage::Controller,
|
|
||||||
] {
|
] {
|
||||||
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||||
fixture.fail(stage);
|
fixture.fail(stage);
|
||||||
@@ -474,7 +740,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result.state,
|
result.state,
|
||||||
server_api::WorkerRestoreState::RolledBack,
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
"{result:?}"
|
"{result:?}"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
@@ -488,12 +754,11 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer
|
|||||||
.iter()
|
.iter()
|
||||||
.all(|sessions| sessions.aliases().is_empty())
|
.all(|sessions| sessions.aliases().is_empty())
|
||||||
);
|
);
|
||||||
fixture.assert_released();
|
|
||||||
fixture.assert_closed_deferred_sessions(&pending);
|
fixture.assert_closed_deferred_sessions(&pending);
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack"
|
"restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=ReconciliationRequired"
|
||||||
);
|
);
|
||||||
fixture.fresh_restore_and_old_replay(request, result);
|
fixture.resume_same_operation(request, pending.request.operation_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -585,6 +850,7 @@ async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reuse
|
|||||||
FailedRestoreResources {
|
FailedRestoreResources {
|
||||||
operation_id: current.operation_id,
|
operation_id: current.operation_id,
|
||||||
sessions: Vec::new(),
|
sessions: Vec::new(),
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
allocation: None,
|
allocation: None,
|
||||||
feature_tasks: None,
|
feature_tasks: None,
|
||||||
},
|
},
|
||||||
@@ -715,14 +981,14 @@ fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconcile
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result.state,
|
result.state,
|
||||||
server_api::WorkerRestoreState::RolledBack,
|
server_api::WorkerRestoreState::Accepted,
|
||||||
"{result:?}"
|
"{result:?}"
|
||||||
);
|
);
|
||||||
fixture.assert_released();
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1"
|
"restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1"
|
||||||
);
|
);
|
||||||
fixture.fresh_restore_and_old_replay(request, result);
|
fixture.resume_same_operation(request, pending.request.operation_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
@@ -745,7 +1011,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result.state,
|
result.state,
|
||||||
server_api::WorkerRestoreState::RolledBack,
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
"{result:?}"
|
"{result:?}"
|
||||||
);
|
);
|
||||||
let diagnostic = serde_json::to_string(&result).unwrap();
|
let diagnostic = serde_json::to_string(&result).unwrap();
|
||||||
@@ -757,8 +1023,12 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
|||||||
// This case deliberately exercises the real Unix socket path-limit error;
|
// This case deliberately exercises the real Unix socket path-limit error;
|
||||||
// positive restore cases retain the factory's normal InProcess transport.
|
// positive restore cases retain the factory's normal InProcess transport.
|
||||||
// Short fixture roots must not accidentally turn it into a successful bind.
|
// Short fixture roots must not accidentally turn it into a successful bind.
|
||||||
let operation_id =
|
let pending = fixture
|
||||||
fixture.backend.completed_restore_failures.lock().unwrap()[&fixture.worker.worker_ref].0;
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let operation_id = pending.request.operation_id;
|
||||||
let socket = fixture
|
let socket = fixture
|
||||||
.backend
|
.backend
|
||||||
.factory
|
.factory
|
||||||
@@ -767,7 +1037,25 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
|||||||
.join("worker.sock");
|
.join("worker.sock");
|
||||||
assert!(socket.as_os_str().as_encoded_bytes().len() >= 108);
|
assert!(socket.as_os_str().as_encoded_bytes().len() >= 108);
|
||||||
assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string()));
|
assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string()));
|
||||||
fixture.assert_released();
|
assert!(!socket.exists());
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.completed_restore_failures
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
fixture
|
fixture
|
||||||
.backend
|
.backend
|
||||||
.factory
|
.factory
|
||||||
@@ -777,7 +1065,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
|
|||||||
eprintln!(
|
eprintln!(
|
||||||
"restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released"
|
"restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released"
|
||||||
);
|
);
|
||||||
fixture.fresh_restore_and_old_replay(request, result);
|
fixture.resume_same_operation(request, operation_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1178,6 +1466,8 @@ fn profile_restore_missing_committed_system_prompt_rolls_back_and_replays_classi
|
|||||||
"#,
|
"#,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
let mut manifest = manifest;
|
||||||
|
manifest.worker.name = worker_name.clone();
|
||||||
metadata.resolved_manifest_snapshot =
|
metadata.resolved_manifest_snapshot =
|
||||||
Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
|
Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
|
||||||
metadata_store.write(&metadata).unwrap();
|
metadata_store.write(&metadata).unwrap();
|
||||||
|
|||||||
@@ -3676,8 +3676,12 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
||||||
|
crate::restore_test_environment::require_restore_test_environment();
|
||||||
let repo = create_clean_repo();
|
let repo = create_clean_repo();
|
||||||
let runtime_root = tempfile::tempdir().unwrap();
|
// Both ends use a short reference into this TempDir-owned tree. Keep
|
||||||
|
// real agent/broker sockets and the existing bind/cleanup assertions.
|
||||||
|
let runtime_root = tempfile::tempdir_in(".").unwrap();
|
||||||
|
let runtime_path = Path::new(runtime_root.path().file_name().unwrap());
|
||||||
let key_root = tempfile::tempdir().unwrap();
|
let key_root = tempfile::tempdir().unwrap();
|
||||||
let key_path = key_root.path().join("id_ed25519");
|
let key_path = key_root.path().join("id_ed25519");
|
||||||
let fallback_key_path = key_root.path().join("id_ed25519_fallback");
|
let fallback_key_path = key_root.path().join("id_ed25519_fallback");
|
||||||
@@ -3689,7 +3693,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(status.success());
|
assert!(status.success());
|
||||||
}
|
}
|
||||||
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
let materializer = RuntimeGitMaterializer::new(runtime_path);
|
||||||
let mut request = request(repo.path());
|
let mut request = request(repo.path());
|
||||||
request.materialization = Some(crate::catalog::RepositoryMaterializationContext {
|
request.materialization = Some(crate::catalog::RepositoryMaterializationContext {
|
||||||
workspace_id: "workspace-1".to_string(),
|
workspace_id: "workspace-1".to_string(),
|
||||||
@@ -3731,7 +3735,7 @@ mod tests {
|
|||||||
kind: server_api::RepositorySourceKind::Ssh,
|
kind: server_api::RepositorySourceKind::Ssh,
|
||||||
uri: "ssh://git@example.test/repo.git".to_string(),
|
uri: "ssh://git@example.test/repo.git".to_string(),
|
||||||
};
|
};
|
||||||
let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation)
|
let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(!command_access.root.join("identity").exists());
|
assert!(!command_access.root.join("identity").exists());
|
||||||
@@ -3873,18 +3877,17 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
// Restore preflight must validate without consuming this one-shot SSH
|
// Restore preflight must validate without consuming this one-shot SSH
|
||||||
// delivery or creating agent/session resources needed by actual bind.
|
// delivery or creating agent/session resources needed by actual bind.
|
||||||
let record_before =
|
let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap();
|
||||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap();
|
|
||||||
materializer
|
materializer
|
||||||
.preflight_bind_working_directory(&id, None)
|
.preflight_bind_working_directory(&id, None)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
|
fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
|
||||||
record_before
|
record_before
|
||||||
);
|
);
|
||||||
assert_eq!(materializer.list_working_directories().unwrap().len(), 1);
|
assert_eq!(materializer.list_working_directories().unwrap().len(), 1);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||||
.map(|entries| entries.count())
|
.map(|entries| entries.count())
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
0
|
0
|
||||||
@@ -3898,7 +3901,7 @@ mod tests {
|
|||||||
assert!(ssh_policy.contains("__repository-ssh"));
|
assert!(ssh_policy.contains("__repository-ssh"));
|
||||||
assert!(!ssh_policy.contains("SSH_AUTH_SOCK"));
|
assert!(!ssh_policy.contains("SSH_AUTH_SOCK"));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||||
.map(|entries| entries.count())
|
.map(|entries| entries.count())
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
1
|
1
|
||||||
@@ -3916,7 +3919,7 @@ mod tests {
|
|||||||
"consumed delivery expiry must not revoke a live Workdir session"
|
"consumed delivery expiry must not revoke a live Workdir session"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||||
.map(|entries| entries.count())
|
.map(|entries| entries.count())
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
1,
|
1,
|
||||||
@@ -3925,7 +3928,7 @@ mod tests {
|
|||||||
drop(binding);
|
drop(binding);
|
||||||
assert!(!ssh_command.exists());
|
assert!(!ssh_command.exists());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||||
.map(|entries| entries.count())
|
.map(|entries| entries.count())
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
0
|
0
|
||||||
@@ -3971,7 +3974,7 @@ mod tests {
|
|||||||
],
|
],
|
||||||
&read_only_command_policy,
|
&read_only_command_policy,
|
||||||
));
|
));
|
||||||
let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR))
|
let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.filter_map(Result::ok)
|
.filter_map(Result::ok)
|
||||||
.map(|entry| entry.path().join("b.sock"))
|
.map(|entry| entry.path().join("b.sock"))
|
||||||
@@ -4104,7 +4107,7 @@ mod tests {
|
|||||||
.code,
|
.code,
|
||||||
"working_directory_repository_access_expired"
|
"working_directory_repository_access_expired"
|
||||||
);
|
);
|
||||||
let restored = RuntimeGitMaterializer::new(runtime_root.path());
|
let restored = RuntimeGitMaterializer::new(runtime_path);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
restored.bind_working_directory(&id, None).unwrap_err().code,
|
restored.bind_working_directory(&id, None).unwrap_err().code,
|
||||||
"working_directory_remote_repository_access_required"
|
"working_directory_remote_repository_access_required"
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ use crate::PromptCatalogSource;
|
|||||||
use crate::controller::{
|
use crate::controller::{
|
||||||
ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle,
|
ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle,
|
||||||
};
|
};
|
||||||
|
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
|
||||||
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
||||||
use manifest::WorkerManifest;
|
use manifest::WorkerManifest;
|
||||||
|
|
||||||
@@ -168,6 +169,38 @@ where
|
|||||||
Ok(PreparedWorker::new(worker, self.layout, self.transport))
|
Ok(PreparedWorker::new(worker, self.layout, self.transport))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reconstruct a saved materialized Worker without allocation, attribution
|
||||||
|
/// repair, Feature installation, or controller exposure. The caller owns the
|
||||||
|
/// later controller layout/transport and must activate before starting it.
|
||||||
|
pub async fn prepare_restored_candidate(
|
||||||
|
self,
|
||||||
|
worker_name: &str,
|
||||||
|
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
|
||||||
|
let mut candidate =
|
||||||
|
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
|
||||||
|
worker_name,
|
||||||
|
self.manifest,
|
||||||
|
self.store,
|
||||||
|
self.prompt_catalog,
|
||||||
|
self.workspace_context,
|
||||||
|
self.filesystem_authority,
|
||||||
|
self.model_client,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
if let Some(workdir_session) = self.workdir_session {
|
||||||
|
candidate
|
||||||
|
.worker_mut()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?
|
||||||
|
.bind_single_workdir_session(Some(workdir_session));
|
||||||
|
}
|
||||||
|
candidate
|
||||||
|
.worker_mut()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?
|
||||||
|
.bind_subjektiv_host(self.subjektiv_host);
|
||||||
|
Ok(candidate)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn prepare_restored(
|
pub async fn prepare_restored(
|
||||||
self,
|
self,
|
||||||
worker_name: &str,
|
worker_name: &str,
|
||||||
|
|||||||
@@ -689,6 +689,35 @@ pub enum WorkerControllerTransport {
|
|||||||
pub struct WorkerController;
|
pub struct WorkerController;
|
||||||
|
|
||||||
impl WorkerController {
|
impl WorkerController {
|
||||||
|
/// Explicit recovery of an unmaterialized Session head. This installs the
|
||||||
|
/// saved Feature composition needed to produce the seed, but creates no
|
||||||
|
/// Controller, input channel, socket, observer, or queue consumer. The caller
|
||||||
|
/// owns the same operation's Feature/session/allocation cleanup on every exit.
|
||||||
|
pub async fn recover_pending_head<C, St>(
|
||||||
|
worker: &mut Worker<C, St>,
|
||||||
|
runtime_base: &Path,
|
||||||
|
) -> std::io::Result<()>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
if worker.needs_initial_session_head_materialization() {
|
||||||
|
worker
|
||||||
|
.validate_subjektiv_body_publication()
|
||||||
|
.map_err(std::io::Error::other)?;
|
||||||
|
let registry = SpawnedWorkerRegistry::new_internal(
|
||||||
|
worker.manifest().worker.name.clone(),
|
||||||
|
worker.scope().clone(),
|
||||||
|
);
|
||||||
|
register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?;
|
||||||
|
worker
|
||||||
|
.materialize_durable_session_head()
|
||||||
|
.await
|
||||||
|
.map_err(std::io::Error::other)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn spawn<C, St>(
|
pub async fn spawn<C, St>(
|
||||||
worker: Worker<C, St>,
|
worker: Worker<C, St>,
|
||||||
runtime_base: &Path,
|
runtime_base: &Path,
|
||||||
|
|||||||
@@ -1066,7 +1066,8 @@ mod tests {
|
|||||||
use tokio::net::UnixListener;
|
use tokio::net::UnixListener;
|
||||||
|
|
||||||
use crate::runtime::dir::RuntimeDir;
|
use crate::runtime::dir::RuntimeDir;
|
||||||
use crate::runtime::worker_allocation::test_util::RuntimeDirSandbox;
|
|
||||||
|
use crate::restore_test_environment::require_restore_test_environment;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct FailTargetPeerStore {
|
struct FailTargetPeerStore {
|
||||||
@@ -1110,11 +1111,13 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test(flavor = "current_thread")]
|
#[tokio::test(flavor = "current_thread")]
|
||||||
async fn state_backed_visibility_and_restore_planning() {
|
async fn state_backed_visibility_and_restore_planning() {
|
||||||
let root = TempDir::new().unwrap();
|
require_restore_test_environment();
|
||||||
|
let root = tempfile::tempdir_in(".").unwrap();
|
||||||
let store_dir = root.path().join("store");
|
let store_dir = root.path().join("store");
|
||||||
let runtime_base = root.path().join("runtime");
|
// Bind, durable child metadata, and discovery probes all use this same
|
||||||
|
// short reference into the tree still owned by root. No cwd mutation.
|
||||||
|
let runtime_base = Path::new(root.path().file_name().unwrap()).join("runtime");
|
||||||
std::fs::create_dir_all(&runtime_base).unwrap();
|
std::fs::create_dir_all(&runtime_base).unwrap();
|
||||||
let _runtime_sandbox = RuntimeDirSandbox::new(&runtime_base);
|
|
||||||
|
|
||||||
let store = FsWorkerStore::new(&store_dir).unwrap();
|
let store = FsWorkerStore::new(&store_dir).unwrap();
|
||||||
let session_id = new_session_id();
|
let session_id = new_session_id();
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ mod metadata_lifecycle;
|
|||||||
pub mod model_client;
|
pub mod model_client;
|
||||||
mod paste_artifact_tool;
|
mod paste_artifact_tool;
|
||||||
pub mod prompt;
|
pub mod prompt;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
pub mod runtime;
|
pub mod runtime;
|
||||||
pub mod runtime_command;
|
pub mod runtime_command;
|
||||||
pub mod segment_log_sink;
|
pub mod segment_log_sink;
|
||||||
|
|||||||
+1004
-30
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,4 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
|
|||||||
// Helpers
|
// Helpers
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
|
use restore_test_environment::require_restore_test_environment;
|
||||||
|
|
||||||
|
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
|
||||||
|
require_restore_test_environment();
|
||||||
|
let temp = tempfile::tempdir_in(".").unwrap();
|
||||||
|
// The relative reference and the TempDir name the same owned tree, without
|
||||||
|
// putting the checkout prefix into sockaddr_un or changing process cwd.
|
||||||
|
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||||
|
(temp, runtime_base)
|
||||||
|
}
|
||||||
|
|
||||||
fn simple_text_events() -> Vec<LlmEvent> {
|
fn simple_text_events() -> Vec<LlmEvent> {
|
||||||
vec![
|
vec![
|
||||||
LlmEvent::text_block_start(0),
|
LlmEvent::text_block_start(0),
|
||||||
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
|
|||||||
manifest: WorkerManifest,
|
manifest: WorkerManifest,
|
||||||
workspace_context: WorkerWorkspaceContext,
|
workspace_context: WorkerWorkspaceContext,
|
||||||
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
||||||
|
require_restore_test_environment();
|
||||||
let store_tmp = tempfile::tempdir().unwrap();
|
let store_tmp = tempfile::tempdir().unwrap();
|
||||||
let store = CombinedStore::new(
|
let store = CombinedStore::new(
|
||||||
FsStore::new(store_tmp.path()).unwrap(),
|
FsStore::new(store_tmp.path()).unwrap(),
|
||||||
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let (tmp, runtime_base) = owned_relative_runtime_base();
|
||||||
let runtime_base = tmp.path().to_owned();
|
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||||
std::mem::forget(tmp);
|
|
||||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
// Retain ownership until controller shutdown (or test-runtime teardown).
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let _ = shutdown_rx.await;
|
||||||
|
drop(tmp);
|
||||||
|
});
|
||||||
handle
|
handle
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let (root, runtime_base) = owned_relative_runtime_base();
|
||||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||||
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
||||||
let store = CombinedStore::new(
|
let store = CombinedStore::new(
|
||||||
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
|
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut rx = handle.subscribe();
|
let mut rx = handle.subscribe();
|
||||||
@@ -1761,11 +1779,12 @@ permission = "write"
|
|||||||
"#;
|
"#;
|
||||||
let client = MockClient::new(simple_text_events());
|
let client = MockClient::new(simple_text_events());
|
||||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let (_tmp, runtime_base) = owned_relative_runtime_base();
|
||||||
let result = WorkerController::spawn(worker, tmp.path()).await;
|
let result = WorkerController::spawn(worker, &runtime_base).await;
|
||||||
assert!(
|
assert!(
|
||||||
result.is_ok(),
|
result.is_ok(),
|
||||||
"feature exposure must not imply delegation authority"
|
"feature exposure must not imply delegation authority: {:?}",
|
||||||
|
result.as_ref().err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ pub mod repositories;
|
|||||||
pub mod repository_access;
|
pub mod repository_access;
|
||||||
pub mod repository_source;
|
pub mod repository_source;
|
||||||
pub mod resource_broker;
|
pub mod resource_broker;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
pub mod retention;
|
pub mod retention;
|
||||||
pub mod runtime_settings;
|
pub mod runtime_settings;
|
||||||
pub mod runtime_subscription;
|
pub mod runtime_subscription;
|
||||||
|
|||||||
@@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{
|
|||||||
WorkerRuntimeExecutionBackend,
|
WorkerRuntimeExecutionBackend,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub(super) fn connected_restore_tempdir() -> tempfile::TempDir {
|
||||||
|
crate::restore_test_environment::require_restore_test_environment();
|
||||||
|
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.parent()
|
||||||
|
.unwrap();
|
||||||
|
tempfile::Builder::new()
|
||||||
|
.prefix("c")
|
||||||
|
.rand_bytes(3)
|
||||||
|
.tempdir_in(checkout.join("target"))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
const WATCHDOG: Duration = Duration::from_secs(45);
|
const WATCHDOG: Duration = Duration::from_secs(45);
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -82,27 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory {
|
|||||||
) -> std::result::Result<(), String> {
|
) -> std::result::Result<(), String> {
|
||||||
self.inner.preflight_restore(request).await
|
self.inner.preflight_restore(request).await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn recover_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||||
|
self.inner.recover_restore_source(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<
|
||||||
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let worker_ref = request.worker_ref.clone();
|
||||||
|
let inner = self.inner.prepare_restore(request).await?;
|
||||||
|
Ok(Box::new(CapturingCandidate {
|
||||||
|
inner,
|
||||||
|
worker_ref,
|
||||||
|
captures: self.captures.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_failed_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.cleanup_failed_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_stopped_worker(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_restored_controller(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
workspace_id: Option<&str>,
|
||||||
|
handle: &worker::WorkerHandle,
|
||||||
|
) {
|
||||||
|
self.inner
|
||||||
|
.activate_restored_controller(worker_ref, workspace_id, handle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CapturingCandidate {
|
||||||
|
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_ref: worker_runtime::identity::WorkerRef,
|
||||||
|
captures: Arc<Captures>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
self.inner.initial_state()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
) -> std::result::Result<
|
) -> std::result::Result<
|
||||||
RuntimeWorkerController,
|
RuntimeWorkerController,
|
||||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
> {
|
> {
|
||||||
let worker_ref = request.worker_ref.clone();
|
let controller = self.inner.activate().await?;
|
||||||
let controller = self.inner.restore_controller(request).await?;
|
|
||||||
self.captures
|
self.captures
|
||||||
.handles
|
.handles
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push((worker_ref.clone(), controller.handle.clone()));
|
.push((self.worker_ref.clone(), controller.handle.clone()));
|
||||||
self.captures
|
self.captures
|
||||||
.restored_clients
|
.restored_clients
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push((worker_ref, controller.workspace_client.clone()));
|
.push((self.worker_ref.clone(), controller.workspace_client.clone()));
|
||||||
Ok(controller)
|
Ok(controller)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.discard().await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Resources {
|
struct Resources {
|
||||||
@@ -262,7 +340,7 @@ async fn call(tools: &BTreeMap<String, Arc<dyn Tool>>, name: &str, args: Value)
|
|||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let root = connected_restore_tempdir();
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
let mut config = test_server_config(root.path());
|
let mut config = test_server_config(root.path());
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -490,21 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
|
|||||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<
|
||||||
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
Ok(Box::new(AttentionRestoreCandidate {
|
||||||
|
inputs: Some((
|
||||||
|
AttentionWorkerFactory {
|
||||||
|
root: self.root.clone(),
|
||||||
|
},
|
||||||
|
request.worker_ref.worker_id.to_string(),
|
||||||
|
)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preparation owns only fixture inputs; reconstruction and Controller startup
|
||||||
|
// remain lazy so discard never needs to shut down a started Worker.
|
||||||
|
struct AttentionRestoreCandidate {
|
||||||
|
inputs: Option<(AttentionWorkerFactory, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
protocol::WorkerStateSnapshot::initial()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
) -> std::result::Result<
|
) -> std::result::Result<
|
||||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
> {
|
> {
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||||
.await
|
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||||
.map_err(|_| {
|
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||||
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
"attention fixture candidate already consumed",
|
||||||
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
)
|
||||||
"attention fixture reconstruction failed",
|
})?;
|
||||||
)
|
factory.controller(worker_id, true).await.map_err(|_| {
|
||||||
})
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"attention fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inputs.take();
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -611,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() {
|
|||||||
|
|
||||||
async fn real_attention_worker_start_restore(lose_response: bool) {
|
async fn real_attention_worker_start_restore(lose_response: bool) {
|
||||||
use session_store::{LogEntry, Store, WorkerSessionStore};
|
use session_store::{LogEntry, Store, WorkerSessionStore};
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
init_clean_git_workspace(dir.path());
|
init_clean_git_workspace(dir.path());
|
||||||
let config = test_server_config(dir.path());
|
let config = test_server_config(dir.path());
|
||||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||||
|
|||||||
@@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() {
|
|||||||
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
||||||
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
||||||
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
||||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str());
|
let path = format!(
|
||||||
let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",
|
||||||
assert_eq!(page["items"].as_array().unwrap().len(),1);
|
ticket.id.as_str()
|
||||||
assert_eq!(page["items"][0]["model_processing_confirmed"],false);
|
);
|
||||||
|
let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||||
|
assert_eq!(page["items"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(page["items"][0]["model_processing_confirmed"], false);
|
||||||
assert!(page["items"][0].get("content").is_none());
|
assert!(page["items"][0].get("content").is_none());
|
||||||
let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||||
assert_eq!(again,page);
|
assert_eq!(again, page);
|
||||||
let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap();
|
let response = build_router(api.clone())
|
||||||
assert_eq!(response.status(),StatusCode::UNAUTHORIZED);
|
.oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap())
|
||||||
delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await;
|
.await
|
||||||
delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await;
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
delivery_http(
|
||||||
|
&api,
|
||||||
|
"GET",
|
||||||
|
"/api/w/other-workspace/notification-diagnostics",
|
||||||
|
None,
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
delivery_http(
|
||||||
|
&api,
|
||||||
|
"GET",
|
||||||
|
&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),
|
||||||
|
None,
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delivery_assignment_path(ticket: &str) -> String {
|
fn delivery_assignment_path(ticket: &str) -> String {
|
||||||
@@ -828,21 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
|
|||||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: delivery_execution::WorkerExecutionRestoreRequest,
|
request: delivery_execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<
|
||||||
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
Ok(Box::new(DeliveryRestoreCandidate {
|
||||||
|
inputs: Some((
|
||||||
|
DeliveryWorkerFactory {
|
||||||
|
root: self.root.clone(),
|
||||||
|
client: self.client.clone(),
|
||||||
|
},
|
||||||
|
request.worker_ref.worker_id.to_string(),
|
||||||
|
)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retain the same scripted client/receipt store without opening either store or
|
||||||
|
// starting a Controller until activation. Discard drops only these owned inputs.
|
||||||
|
struct DeliveryRestoreCandidate {
|
||||||
|
inputs: Option<(DeliveryWorkerFactory, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
protocol::WorkerStateSnapshot::initial()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
) -> std::result::Result<
|
) -> std::result::Result<
|
||||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
> {
|
> {
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||||
.await
|
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||||
.map_err(|_| {
|
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||||
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
"delivery fixture candidate already consumed",
|
||||||
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
)
|
||||||
"delivery fixture reconstruction failed",
|
})?;
|
||||||
)
|
factory.controller(worker_id, true).await.map_err(|_| {
|
||||||
})
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"delivery fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inputs.take();
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -941,6 +1002,7 @@ async fn delivery_real_fixture(
|
|||||||
Arc<DeliveryResponseLossBackend>,
|
Arc<DeliveryResponseLossBackend>,
|
||||||
DeliveryScriptedClient,
|
DeliveryScriptedClient,
|
||||||
) {
|
) {
|
||||||
|
let _environment = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
let config = test_server_config(root);
|
let config = test_server_config(root);
|
||||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||||
seed_test_registered_workspace(store.as_ref(), &config)
|
seed_test_registered_workspace(store.as_ref(), &config)
|
||||||
@@ -1146,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
|||||||
"uncertain delivery changed request id or payload: {attempts:?}"
|
"uncertain delivery changed request id or payload: {attempts:?}"
|
||||||
);
|
);
|
||||||
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
||||||
let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery {
|
let unknown = api
|
||||||
assignment_id: Some(assignment.into()), ..Default::default()
|
.store
|
||||||
}).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt");
|
.notification_diagnostics(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&server_api::NotificationDiagnosticsQuery {
|
||||||
|
assignment_id: Some(assignment.into()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.items
|
||||||
|
.into_iter()
|
||||||
|
.find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown)
|
||||||
|
.expect(
|
||||||
|
"saved unknown acceptance must remain visible while Runtime already has its receipt",
|
||||||
|
);
|
||||||
assert!(!unknown.model_processing_confirmed);
|
assert!(!unknown.model_processing_confirmed);
|
||||||
// Change mutable rendering context while the acknowledgement is unknown.
|
// Change mutable rendering context while the acknowledgement is unknown.
|
||||||
// A new self-report is not a new input and cannot change the frozen retry.
|
// A new self-report is not a new input and cannot change the frozen retry.
|
||||||
@@ -1209,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
|||||||
}
|
}
|
||||||
delivery_wait_idle(&reopened, &worker, 2).await;
|
delivery_wait_idle(&reopened, &worker, 2).await;
|
||||||
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
||||||
let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await;
|
let evidence = delivery_http(
|
||||||
assert_eq!(evidence["items"][0]["stage"],"runtime_accepted");
|
&reopened,
|
||||||
assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id));
|
"GET",
|
||||||
assert_eq!(evidence["items"][0]["model_processing_confirmed"],false);
|
&format!(
|
||||||
assert_eq!(evidence["items"][0]["retry_candidate"],false);
|
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",
|
||||||
|
unknown.source_id
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
StatusCode::OK,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(evidence["items"][0]["stage"], "runtime_accepted");
|
||||||
|
assert_eq!(
|
||||||
|
evidence["items"][0]["delivery_id"],
|
||||||
|
json!(unknown.delivery_id)
|
||||||
|
);
|
||||||
|
assert_eq!(evidence["items"][0]["model_processing_confirmed"], false);
|
||||||
|
assert_eq!(evidence["items"][0]["retry_candidate"], false);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
delivery_commits(&reopened, &worker)
|
delivery_commits(&reopened, &worker)
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -187,7 +187,14 @@ async fn config_call(
|
|||||||
runtime
|
runtime
|
||||||
.invoke(
|
.invoke(
|
||||||
path.into(),
|
path.into(),
|
||||||
if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)},
|
if matches!(operation, "edit" | "write") {
|
||||||
|
wip_protocol::InterfaceReference {
|
||||||
|
scope: "/".into(),
|
||||||
|
name: format!("yoi.file/{operation}-text/v1"),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
config_interface(path)
|
||||||
|
},
|
||||||
operation.into(),
|
operation.into(),
|
||||||
args,
|
args,
|
||||||
agen::tool::ToolExecutionContext::direct(),
|
agen::tool::ToolExecutionContext::direct(),
|
||||||
@@ -581,14 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
|
|||||||
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||||
Ok(controller)
|
Ok(controller)
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn preflight_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.preflight_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recover_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||||
|
self.inner.recover_restore_source(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<
|
||||||
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let inner = self.inner.prepare_restore(request).await?;
|
||||||
|
Ok(Box::new(ConfigCaptureCandidate {
|
||||||
|
inner,
|
||||||
|
client: self.client.clone(),
|
||||||
|
controller_abort: self.controller_abort.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_failed_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.cleanup_failed_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_stopped_worker(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_restored_controller(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
workspace_id: Option<&str>,
|
||||||
|
handle: &worker::WorkerHandle,
|
||||||
|
) {
|
||||||
|
self.inner
|
||||||
|
.activate_restored_controller(worker_ref, workspace_id, handle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ConfigCaptureCandidate {
|
||||||
|
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
client: Arc<Mutex<Option<Arc<dyn worker::WorkspaceClient>>>>,
|
||||||
|
controller_abort: Arc<Mutex<Option<tokio::task::AbortHandle>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
self.inner.initial_state()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
) -> std::result::Result<
|
) -> std::result::Result<
|
||||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
> {
|
> {
|
||||||
self.inner.restore_controller(request).await
|
let controller = self.inner.activate().await?;
|
||||||
|
*self.client.lock().unwrap() = Some(controller.workspace_client.clone());
|
||||||
|
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||||
|
Ok(controller)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.discard().await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -717,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
|||||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
||||||
decode_runtime_request_source_claims,
|
decode_runtime_request_source_claims,
|
||||||
};
|
};
|
||||||
|
let root = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
||||||
let mut fixture = manual_worker_assignment_fixture().await;
|
let mut fixture = manual_worker_assignment_fixture().await;
|
||||||
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
||||||
@@ -796,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
|||||||
|
|
||||||
// Actual embedded Runtime factory -> real Worker controller -> captured
|
// Actual embedded Runtime factory -> real Worker controller -> captured
|
||||||
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
||||||
let root = tempfile::tempdir().unwrap();
|
|
||||||
init_clean_git_workspace(root.path());
|
init_clean_git_workspace(root.path());
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let addr = listener.local_addr().unwrap();
|
let addr = listener.local_addr().unwrap();
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"tasks": {
|
"tasks": {
|
||||||
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts",
|
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts",
|
||||||
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
|
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
|
||||||
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts"
|
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts"
|
||||||
},
|
},
|
||||||
"fmt": {
|
"fmt": {
|
||||||
"proseWrap": "never"
|
"proseWrap": "never"
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
//! Fail-closed preflight shared by restore fixtures across Worker, Runtime and Server.
|
||||||
|
//!
|
||||||
|
//! The normal runner supplies a separate namespace per process; a manually
|
||||||
|
//! configured invocation may use the checkout-owned base itself. This module
|
||||||
|
//! never sets process-wide environment or creates directories/tables.
|
||||||
|
|
||||||
|
use std::fs;
|
||||||
|
use std::path::{Component, Path, PathBuf};
|
||||||
|
|
||||||
|
fn validate_restore_test_paths(
|
||||||
|
base: &Path,
|
||||||
|
resolved: [Option<PathBuf>; 5],
|
||||||
|
) -> Result<PathBuf, &'static str> {
|
||||||
|
let root = resolved[0]
|
||||||
|
.as_deref()
|
||||||
|
.filter(|config| config.file_name().is_some_and(|name| name == "config"))
|
||||||
|
.and_then(Path::parent)
|
||||||
|
.ok_or("config")?;
|
||||||
|
let namespace = root.strip_prefix(base).map_err(|_| "config")?;
|
||||||
|
if namespace
|
||||||
|
.components()
|
||||||
|
.any(|part| !matches!(part, Component::Normal(_)))
|
||||||
|
{
|
||||||
|
return Err("config");
|
||||||
|
}
|
||||||
|
for ((label, suffix), actual) in [
|
||||||
|
("config", "config"),
|
||||||
|
("data", "data"),
|
||||||
|
("secrets", "data"),
|
||||||
|
("runtime", "run"),
|
||||||
|
("allocation", "run/workers.json"),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.zip(resolved.iter())
|
||||||
|
{
|
||||||
|
if actual.as_deref() != Some(root.join(suffix).as_path()) {
|
||||||
|
return Err(label);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(root.to_path_buf())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_owned_directory(checkout: &Path, directory: &Path) {
|
||||||
|
let mut checked = checkout.to_path_buf();
|
||||||
|
for part in directory.strip_prefix(checkout).unwrap().components() {
|
||||||
|
assert!(matches!(part, Component::Normal(_)));
|
||||||
|
checked.push(part);
|
||||||
|
let metadata = fs::symlink_metadata(&checked)
|
||||||
|
.expect("create isolated restore test directories before launching the test process");
|
||||||
|
assert!(
|
||||||
|
metadata.is_dir() && !metadata.file_type().is_symlink(),
|
||||||
|
"restore test directory must not traverse symlinks"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn require_restore_test_environment() {
|
||||||
|
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.parent()
|
||||||
|
.unwrap();
|
||||||
|
let base = checkout.join("target/restore-test-environment");
|
||||||
|
// Compare all resolver results before inspecting even an absent allocation
|
||||||
|
// table (its lookup can create a lock). Never inspect a live fallback path.
|
||||||
|
let root = validate_restore_test_paths(
|
||||||
|
&base,
|
||||||
|
[
|
||||||
|
manifest::paths::config_dir(),
|
||||||
|
manifest::paths::data_dir(),
|
||||||
|
manifest::paths::secret_data_dir(),
|
||||||
|
manifest::paths::runtime_dir(),
|
||||||
|
manifest::paths::worker_allocation_path(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap_or_else(|label| {
|
||||||
|
panic!(
|
||||||
|
"restore fixture requires isolated {label}: run deno task test, or configure YOI_CONFIG_DIR=<checkout>/target/restore-test-environment/config, YOI_DATA_DIR=<checkout>/target/restore-test-environment/data, YOI_RUNTIME_DIR=<checkout>/target/restore-test-environment/run before launch"
|
||||||
|
)
|
||||||
|
});
|
||||||
|
assert_eq!(fs::canonicalize(checkout).unwrap(), checkout);
|
||||||
|
for directory in [
|
||||||
|
root.clone(),
|
||||||
|
root.join("config"),
|
||||||
|
root.join("data"),
|
||||||
|
root.join("run"),
|
||||||
|
] {
|
||||||
|
require_owned_directory(checkout, &directory);
|
||||||
|
}
|
||||||
|
for name in ["workers.json", "workers.json.lock"] {
|
||||||
|
match fs::symlink_metadata(root.join("run").join(name)) {
|
||||||
|
Ok(metadata) => assert!(
|
||||||
|
metadata.is_file() && !metadata.file_type().is_symlink(),
|
||||||
|
"owned allocation paths must be regular files, not symlinks"
|
||||||
|
),
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||||
|
Err(error) => panic!("inspect owned restore allocation path: {error}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restore_fixture_path_validation_rejects_fallback_before_resource_access() {
|
||||||
|
let base = Path::new("/owned/test-environment");
|
||||||
|
for root in [base.to_path_buf(), base.join("run-1/worker-shard-2")] {
|
||||||
|
let safe = ["config", "data", "data", "run", "run/workers.json"]
|
||||||
|
.map(|suffix| Some(root.join(suffix)));
|
||||||
|
assert_eq!(
|
||||||
|
validate_restore_test_paths(base, safe.clone()),
|
||||||
|
Ok(root.clone())
|
||||||
|
);
|
||||||
|
for index in 0..safe.len() {
|
||||||
|
for fallback in [None, Some(PathBuf::from("/not-a-fixture"))] {
|
||||||
|
let mut unsafe_paths = safe.clone();
|
||||||
|
unsafe_paths[index] = fallback;
|
||||||
|
assert!(validate_restore_test_paths(base, unsafe_paths).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for root in [
|
||||||
|
base.join("../escape"),
|
||||||
|
base.join("safe/../../escape"),
|
||||||
|
PathBuf::from("/outside"),
|
||||||
|
] {
|
||||||
|
let paths = ["config", "data", "data", "run", "run/workers.json"]
|
||||||
|
.map(|suffix| Some(root.join(suffix)));
|
||||||
|
assert!(validate_restore_test_paths(base, paths).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
+367
-2
@@ -6,6 +6,7 @@ import {
|
|||||||
mapConcurrent,
|
mapConcurrent,
|
||||||
parseJobs,
|
parseJobs,
|
||||||
parseRustListing,
|
parseRustListing,
|
||||||
|
prepareTestIsolation,
|
||||||
resultSummary,
|
resultSummary,
|
||||||
rustTestCommands,
|
rustTestCommands,
|
||||||
shardRustCommand,
|
shardRustCommand,
|
||||||
@@ -202,6 +203,39 @@ Deno.test("round-robin shards retain every listed name exactly once at threshold
|
|||||||
assertThrows(() => shardRustCommand(task, ["same", "same"]));
|
assertThrows(() => shardRustCommand(task, ["same", "same"]));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("global runtime-path packages serialize every normal target without losing names", () => {
|
||||||
|
for (const pkg of ["worker", "worker-runtime", "yoi-workspace-server"]) {
|
||||||
|
for (const target of ["lib", "controller_restore_test"]) {
|
||||||
|
const task = {
|
||||||
|
name: `${pkg}/${target}`,
|
||||||
|
command: "/test-binary",
|
||||||
|
args: [],
|
||||||
|
cwd: "/pkg",
|
||||||
|
};
|
||||||
|
for (const count of [0, 1, 59, 72, 73, 1135]) {
|
||||||
|
const names = listedNames(count);
|
||||||
|
const shards = shardRustCommand(task, names);
|
||||||
|
assert(
|
||||||
|
shards.flatMap((shard) => shard.selectedNames).sort().join() ===
|
||||||
|
[...names].sort().join(),
|
||||||
|
);
|
||||||
|
for (const shard of shards) {
|
||||||
|
const args = shard.task.args;
|
||||||
|
assert(args[args.indexOf("--test-threads") + 1] === "1");
|
||||||
|
if (count > 72) {
|
||||||
|
assert(args[0] === "--exact");
|
||||||
|
assert(args.slice(3).join() === shard.selectedNames.join());
|
||||||
|
} else {
|
||||||
|
assert(args.join() === "--test-threads,1");
|
||||||
|
}
|
||||||
|
assert(!args.includes("--ignored") && !args.includes("--skip"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert(task.args.length === 0, "do not mutate discovered commands");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
Deno.test("scheduler drains active workers before propagating a callback error", async () => {
|
Deno.test("scheduler drains active workers before propagating a callback error", async () => {
|
||||||
let drained = false;
|
let drained = false;
|
||||||
let failed = false;
|
let failed = false;
|
||||||
@@ -266,6 +300,276 @@ function emergencyKill(pid: number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function assertRejects(
|
||||||
|
run: () => Promise<unknown>,
|
||||||
|
message: string,
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
await run();
|
||||||
|
} catch (error) {
|
||||||
|
assert(String(error).includes(message), String(error));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw new Error(`expected rejection containing ${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
Deno.test({
|
||||||
|
name:
|
||||||
|
"isolated real children use bounded disjoint tables and sanitized Git/compiler environments",
|
||||||
|
ignore: Deno.build.os !== "linux",
|
||||||
|
async fn() {
|
||||||
|
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||||
|
const environmentSnapshot = () =>
|
||||||
|
JSON.stringify(
|
||||||
|
Object.entries(process.env).sort(([a], [b]) => a.localeCompare(b)),
|
||||||
|
);
|
||||||
|
const before = environmentSnapshot();
|
||||||
|
try {
|
||||||
|
// An ancestor repository must not make an uninitialized tempfile look like
|
||||||
|
// a repository. No developer repository or configuration is borrowed.
|
||||||
|
for (const dir of [".git/objects", ".git/refs"]) {
|
||||||
|
await Deno.mkdir(`${checkout}/${dir}`, { recursive: true });
|
||||||
|
}
|
||||||
|
await Deno.writeTextFile(
|
||||||
|
`${checkout}/.git/HEAD`,
|
||||||
|
"ref: refs/heads/main\n",
|
||||||
|
);
|
||||||
|
const [first, second] = await Promise.all([
|
||||||
|
prepareTestIsolation(checkout, "same/output/id"),
|
||||||
|
prepareTestIsolation(checkout, "same/output/id"),
|
||||||
|
]);
|
||||||
|
assert(
|
||||||
|
first.root !== second.root,
|
||||||
|
"concurrent runner ids must not collide",
|
||||||
|
);
|
||||||
|
const task = {
|
||||||
|
...fixtureTask("isolation-env", checkout),
|
||||||
|
env: {
|
||||||
|
CARGO_MANIFEST_DIR: `${checkout}/crates/example`,
|
||||||
|
CARGO_HOME: `${checkout}/compiler-cache`,
|
||||||
|
RUSTUP_HOME: `${checkout}/toolchain-cache`,
|
||||||
|
YOI_HOME: "/live/yoi",
|
||||||
|
YOI_CONFIG_DIR: "/live/config",
|
||||||
|
YOI_DATA_DIR: "/live/data",
|
||||||
|
YOI_RUNTIME_DIR: "/live/run",
|
||||||
|
HOME: "/live/home",
|
||||||
|
TMPDIR: "/deleted/compiler-tmp",
|
||||||
|
GIT_DIR: "/live/git",
|
||||||
|
GIT_WORK_TREE: "/live/tree",
|
||||||
|
GIT_CONFIG: "/live/gitconfig",
|
||||||
|
GIT_CONFIG_GLOBAL: "/live/gitconfig",
|
||||||
|
GIT_CONFIG_SYSTEM: "/live/system-gitconfig",
|
||||||
|
GIT_TEMPLATE_DIR: "/live/template",
|
||||||
|
GIT_CONFIG_PARAMETERS: "'core.hooksPath'='/live/hooks'",
|
||||||
|
GIT_CONFIG_COUNT: "1",
|
||||||
|
GIT_CONFIG_KEY_0: "core.hooksPath",
|
||||||
|
GIT_CONFIG_VALUE_0: "/live/hooks",
|
||||||
|
HTTP_PROXY: "http://live.invalid",
|
||||||
|
https_proxy: "http://live.invalid",
|
||||||
|
ALL_PROXY: "http://live.invalid",
|
||||||
|
NO_PROXY: "live.invalid",
|
||||||
|
RUSTC_WRAPPER: "/live/sccache",
|
||||||
|
RUSTC_WORKSPACE_WRAPPER: "/live/wrapper",
|
||||||
|
SCCACHE_DISABLE: "0",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const taskBefore = JSON.stringify(task);
|
||||||
|
const commands = await Promise.all([
|
||||||
|
first.wrapCommand(task, "pkg/a"),
|
||||||
|
first.wrapCommand(task, "pkg?a"), // same sanitized label
|
||||||
|
first.wrapCommand(task, "pkg/a"), // same exact label
|
||||||
|
second.wrapCommand(task, "pkg/a"),
|
||||||
|
]);
|
||||||
|
assert(JSON.stringify(task) === taskBefore);
|
||||||
|
assert(
|
||||||
|
environmentSnapshot() === before,
|
||||||
|
"host environment changed",
|
||||||
|
);
|
||||||
|
const results = await Promise.all(
|
||||||
|
commands.map((command, index) =>
|
||||||
|
execute(command, {
|
||||||
|
output: checkout,
|
||||||
|
label: `environment-${index}`,
|
||||||
|
timeoutMs: 10_000,
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const tables = new Set<string>();
|
||||||
|
for (const [index, result] of results.entries()) {
|
||||||
|
assert(result.code === 0 && !result.error, JSON.stringify(result));
|
||||||
|
const observed = JSON.parse(result.stdout);
|
||||||
|
const env = observed.env as Record<string, string>;
|
||||||
|
assert(observed.cwd === checkout && commands[index].cwd === task.cwd);
|
||||||
|
assert(commands[index].args === task.args);
|
||||||
|
assert(env.CARGO_MANIFEST_DIR === task.env.CARGO_MANIFEST_DIR);
|
||||||
|
assert(env.CARGO_HOME === task.env.CARGO_HOME);
|
||||||
|
assert(env.RUSTUP_HOME === task.env.RUSTUP_HOME);
|
||||||
|
const namespace = env.YOI_RUNTIME_DIR.replace(/\/run$/, "");
|
||||||
|
const prefix = `${checkout}/target/restore-test-environment/`;
|
||||||
|
assert(namespace.startsWith(prefix));
|
||||||
|
const suffix = namespace.slice(prefix.length);
|
||||||
|
assert(
|
||||||
|
suffix.split("/").every((part) => /^[a-zA-Z0-9_-]+$/.test(part)),
|
||||||
|
);
|
||||||
|
assert(env.YOI_CONFIG_DIR === `${namespace}/config`);
|
||||||
|
assert(env.YOI_DATA_DIR === `${namespace}/data`);
|
||||||
|
assert(
|
||||||
|
env.YOI_HOME === env.YOI_DATA_DIR,
|
||||||
|
"secret data must equal data",
|
||||||
|
);
|
||||||
|
assert(env.HOME === `${namespace}/home`);
|
||||||
|
for (
|
||||||
|
const dir of [
|
||||||
|
namespace,
|
||||||
|
env.YOI_CONFIG_DIR,
|
||||||
|
env.YOI_DATA_DIR,
|
||||||
|
env.YOI_RUNTIME_DIR,
|
||||||
|
env.HOME,
|
||||||
|
]
|
||||||
|
) {
|
||||||
|
assert((await Deno.lstat(dir)).isDirectory);
|
||||||
|
assert(await Deno.realPath(dir) === dir);
|
||||||
|
}
|
||||||
|
for (const key of ["TMPDIR", "TMP", "TEMP"]) {
|
||||||
|
assert(env[key] === `${checkout}/target/t`);
|
||||||
|
}
|
||||||
|
for (
|
||||||
|
const key of [
|
||||||
|
"GIT_DIR",
|
||||||
|
"GIT_WORK_TREE",
|
||||||
|
"GIT_CONFIG",
|
||||||
|
"GIT_CONFIG_PARAMETERS",
|
||||||
|
"GIT_CONFIG_COUNT",
|
||||||
|
"GIT_CONFIG_KEY_0",
|
||||||
|
"GIT_CONFIG_VALUE_0",
|
||||||
|
"HTTP_PROXY",
|
||||||
|
"https_proxy",
|
||||||
|
"ALL_PROXY",
|
||||||
|
"NO_PROXY",
|
||||||
|
]
|
||||||
|
) {
|
||||||
|
assert(!(key in env), `${key} leaked to child`);
|
||||||
|
}
|
||||||
|
assert(env.GIT_CONFIG_GLOBAL === "/dev/null");
|
||||||
|
assert(env.GIT_CONFIG_SYSTEM === "/dev/null");
|
||||||
|
assert(env.GIT_CONFIG_NOSYSTEM === "1");
|
||||||
|
assert(env.GIT_CEILING_DIRECTORIES === checkout);
|
||||||
|
assert(env.GIT_TEMPLATE_DIR === `${namespace}/git-template`);
|
||||||
|
assert(observed.globalConfig === "");
|
||||||
|
assert(observed.ancestorDiscovered === false);
|
||||||
|
assert(observed.hooks.length === 0 && observed.info.length === 0);
|
||||||
|
assert(observed.author.includes("normal-tests@example.invalid"));
|
||||||
|
assert(env.GIT_AUTHOR_NAME === env.GIT_COMMITTER_NAME);
|
||||||
|
assert(env.GIT_AUTHOR_EMAIL === env.GIT_COMMITTER_EMAIL);
|
||||||
|
assert(env.RUSTC_WRAPPER === "" && env.RUSTC_WORKSPACE_WRAPPER === "");
|
||||||
|
assert(env.SCCACHE_DISABLE === "1");
|
||||||
|
const table = `${env.YOI_RUNTIME_DIR}/workers.json`;
|
||||||
|
tables.add(table);
|
||||||
|
assert(await Deno.readTextFile(table) === String(observed.pid));
|
||||||
|
}
|
||||||
|
assert(
|
||||||
|
tables.size === commands.length,
|
||||||
|
"independent commands shared a table",
|
||||||
|
);
|
||||||
|
assert(environmentSnapshot() === before);
|
||||||
|
} finally {
|
||||||
|
await Deno.remove(checkout, { recursive: true });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
Deno.test("isolation refuses lexical aliases and symlinked owned paths before external writes", async () => {
|
||||||
|
for (
|
||||||
|
const unsafe of ["target", "target/restore-test-environment", "target/t"]
|
||||||
|
) {
|
||||||
|
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||||
|
const outside = await Deno.realPath(await Deno.makeTempDir());
|
||||||
|
try {
|
||||||
|
await Deno.writeTextFile(`${outside}/sentinel`, "untouched");
|
||||||
|
if (unsafe !== "target") await Deno.mkdir(`${checkout}/target`);
|
||||||
|
await Deno.symlink(outside, `${checkout}/${unsafe}`);
|
||||||
|
await assertRejects(
|
||||||
|
() => prepareTestIsolation(checkout, "run"),
|
||||||
|
"symlink",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
Array.from(Deno.readDirSync(outside)).map((entry) => entry.name)
|
||||||
|
.join() ===
|
||||||
|
"sentinel",
|
||||||
|
);
|
||||||
|
assert(await Deno.readTextFile(`${outside}/sentinel`) === "untouched");
|
||||||
|
} finally {
|
||||||
|
await Deno.remove(checkout, { recursive: true });
|
||||||
|
await Deno.remove(outside, { recursive: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||||
|
try {
|
||||||
|
await Deno.symlink(checkout, `${checkout}/alias`);
|
||||||
|
await assertRejects(
|
||||||
|
() => prepareTestIsolation(`${checkout}/alias`, "run"),
|
||||||
|
"symlink",
|
||||||
|
);
|
||||||
|
for (const suffix of ["/../escape", "/./child", "//child"]) {
|
||||||
|
await assertRejects(
|
||||||
|
() => prepareTestIsolation(`${checkout}${suffix}`, "run"),
|
||||||
|
"normal-only",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert(
|
||||||
|
Array.from(Deno.readDirSync(checkout)).map((entry) => entry.name)
|
||||||
|
.join() ===
|
||||||
|
"alias",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await Deno.remove(checkout, { recursive: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Deno.test("command wrapping rechecks owned ancestors and refuses symlinked namespace children", async () => {
|
||||||
|
for (
|
||||||
|
const unsafe of [
|
||||||
|
"ancestor",
|
||||||
|
"tmp",
|
||||||
|
"config",
|
||||||
|
"data",
|
||||||
|
"run",
|
||||||
|
"home",
|
||||||
|
"git-template",
|
||||||
|
"git-template/info",
|
||||||
|
"git-template/hooks",
|
||||||
|
]
|
||||||
|
) {
|
||||||
|
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||||
|
const outside = await Deno.realPath(await Deno.makeTempDir());
|
||||||
|
try {
|
||||||
|
const isolation = await prepareTestIsolation(checkout, "run");
|
||||||
|
let path: string;
|
||||||
|
if (unsafe === "ancestor") {
|
||||||
|
path = isolation.root;
|
||||||
|
await Deno.remove(path);
|
||||||
|
} else if (unsafe === "tmp") {
|
||||||
|
path = `${checkout}/target/t`;
|
||||||
|
await Deno.remove(path);
|
||||||
|
} else {
|
||||||
|
const namespace = `${isolation.root}/unsafe-0`;
|
||||||
|
await Deno.mkdir(namespace);
|
||||||
|
if (unsafe.includes("/")) await Deno.mkdir(`${namespace}/git-template`);
|
||||||
|
path = `${namespace}/${unsafe}`;
|
||||||
|
}
|
||||||
|
await Deno.symlink(outside, path);
|
||||||
|
await assertRejects(
|
||||||
|
() => isolation.wrapCommand(fixtureTask("success", checkout), "unsafe"),
|
||||||
|
"symlink",
|
||||||
|
);
|
||||||
|
assert(Array.from(Deno.readDirSync(outside)).length === 0);
|
||||||
|
} finally {
|
||||||
|
await Deno.remove(checkout, { recursive: true });
|
||||||
|
await Deno.remove(outside, { recursive: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) {
|
for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) {
|
||||||
Deno.test({
|
Deno.test({
|
||||||
name: `execute streams real ${mode} child output and records its exit`,
|
name: `execute streams real ${mode} child output and records its exit`,
|
||||||
@@ -436,15 +740,28 @@ Deno.test({
|
|||||||
"runtime listing persists complete name inventory for large and small binaries",
|
"runtime listing persists complete name inventory for large and small binaries",
|
||||||
ignore: Deno.build.os !== "linux",
|
ignore: Deno.build.os !== "linux",
|
||||||
async fn() {
|
async fn() {
|
||||||
const output = await Deno.makeTempDir();
|
const output = await Deno.realPath(await Deno.makeTempDir());
|
||||||
try {
|
try {
|
||||||
|
const isolation = await prepareTestIsolation(output, "discovery");
|
||||||
|
const wrappedTables = new Set<string>();
|
||||||
const discovery = await discoverRustCommands(
|
const discovery = await discoverRustCommands(
|
||||||
[
|
[
|
||||||
fixtureTask("list-large", output),
|
fixtureTask("list-large", output),
|
||||||
fixtureTask("list-small", output),
|
fixtureTask("list-small", output),
|
||||||
],
|
],
|
||||||
2,
|
2,
|
||||||
{ output },
|
{
|
||||||
|
output,
|
||||||
|
run: async (task, label) => {
|
||||||
|
const wrapped = await isolation.wrapCommand(task, label);
|
||||||
|
wrappedTables.add(`${wrapped.env!.YOI_RUNTIME_DIR}/workers.json`);
|
||||||
|
return await execute(wrapped, { output, label, timeoutMs: 10_000 });
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
wrappedTables.size === 2,
|
||||||
|
"all discovery launches must use the callback",
|
||||||
);
|
);
|
||||||
assert(discovery.inventory.complete);
|
assert(discovery.inventory.complete);
|
||||||
assert(
|
assert(
|
||||||
@@ -635,6 +952,54 @@ if (import.meta.main) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
Deno.exit(mode === "list-nonzero" ? 7 : 0);
|
Deno.exit(mode === "list-nonzero" ? 7 : 0);
|
||||||
|
} else if (mode === "isolation-env") {
|
||||||
|
const env = Deno.env.toObject();
|
||||||
|
const scratch = await Deno.makeTempDir({ dir: env.TMPDIR });
|
||||||
|
const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes);
|
||||||
|
async function git(args: string[]) {
|
||||||
|
return await new Deno.Command("git", {
|
||||||
|
args,
|
||||||
|
cwd: scratch,
|
||||||
|
stdout: "piped",
|
||||||
|
stderr: "piped",
|
||||||
|
}).output();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const ownedRepo = await git([
|
||||||
|
"-C",
|
||||||
|
output,
|
||||||
|
"rev-parse",
|
||||||
|
"--show-toplevel",
|
||||||
|
]);
|
||||||
|
assert(
|
||||||
|
ownedRepo.success && text(ownedRepo.stdout).trim() === output,
|
||||||
|
"owned ancestor repository fixture must be discoverable at its root",
|
||||||
|
);
|
||||||
|
const ancestor = await git(["rev-parse", "--show-toplevel"]);
|
||||||
|
const init = await git(["init", "--quiet"]);
|
||||||
|
assert(init.success, text(init.stderr));
|
||||||
|
const global = await git(["config", "--global", "--list"]);
|
||||||
|
assert(global.success, text(global.stderr));
|
||||||
|
const author = await git(["var", "GIT_AUTHOR_IDENT"]);
|
||||||
|
assert(author.success, text(author.stderr));
|
||||||
|
await Deno.writeTextFile(
|
||||||
|
`${env.YOI_RUNTIME_DIR}/workers.json`,
|
||||||
|
String(Deno.pid),
|
||||||
|
);
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
env,
|
||||||
|
cwd: Deno.cwd(),
|
||||||
|
pid: Deno.pid,
|
||||||
|
ancestorDiscovered: ancestor.success,
|
||||||
|
globalConfig: text(global.stdout),
|
||||||
|
author: text(author.stdout),
|
||||||
|
hooks: Array.from(Deno.readDirSync(`${scratch}/.git/hooks`)),
|
||||||
|
info: Array.from(Deno.readDirSync(`${scratch}/.git/info`)),
|
||||||
|
}));
|
||||||
|
} finally {
|
||||||
|
await Deno.remove(scratch, { recursive: true });
|
||||||
|
}
|
||||||
|
Deno.exit(0);
|
||||||
} else if (mode === "runner-parent") {
|
} else if (mode === "runner-parent") {
|
||||||
const cancellation = installCancellationHandlers();
|
const cancellation = installCancellationHandlers();
|
||||||
try {
|
try {
|
||||||
|
|||||||
+197
-19
@@ -1,9 +1,10 @@
|
|||||||
// Bound top-level suites/shards, not total processes; large shards use two test threads.
|
// Bound top-level suites/shards, not total processes; large shards normally use two threads.
|
||||||
// Cargo selects targets/features. Retain all listed names; never skip cached successes.
|
// Cargo selects targets/features. Retain all listed names; never skip cached successes.
|
||||||
import { spawn } from "node:child_process";
|
import { spawn } from "node:child_process";
|
||||||
import process from "node:process";
|
import process from "node:process";
|
||||||
import { availableParallelism } from "node:os";
|
import { availableParallelism } from "node:os";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { posix } from "node:path";
|
||||||
|
|
||||||
// Safety limits, NOT the separate 60s performance goal.
|
// Safety limits, NOT the separate 60s performance goal.
|
||||||
export const SUITE_TIMEOUT_MS = 180_000;
|
export const SUITE_TIMEOUT_MS = 180_000;
|
||||||
@@ -65,7 +66,19 @@ export function shardRustCommand(
|
|||||||
if (new Set(names).size !== names.length) {
|
if (new Set(names).size !== names.length) {
|
||||||
throw new Error("cannot shard duplicate test names");
|
throw new Error("cannot shard duplicate test names");
|
||||||
}
|
}
|
||||||
if (names.length <= 72) return [{ task, selectedNames: [...names] }];
|
// RuntimeDirSandbox changes process-global paths. Do not overlap its writes
|
||||||
|
// with restore guards in the same process, including small integration targets.
|
||||||
|
const serial = ["worker", "worker-runtime", "yoi-workspace-server"].includes(
|
||||||
|
task.name.split("/")[0],
|
||||||
|
);
|
||||||
|
if (names.length <= 72) {
|
||||||
|
return [{
|
||||||
|
task: serial
|
||||||
|
? { ...task, args: [...task.args, "--test-threads", "1"] }
|
||||||
|
: task,
|
||||||
|
selectedNames: [...names],
|
||||||
|
}];
|
||||||
|
}
|
||||||
const count = Math.min(32, Math.ceil(names.length / 36));
|
const count = Math.min(32, Math.ceil(names.length / 36));
|
||||||
const shards: string[][] = Array.from({ length: count }, () => []);
|
const shards: string[][] = Array.from({ length: count }, () => []);
|
||||||
names.forEach((name, index) => shards[index % count].push(name));
|
names.forEach((name, index) => shards[index % count].push(name));
|
||||||
@@ -75,7 +88,13 @@ export function shardRustCommand(
|
|||||||
name: `${task.name}/shard-${
|
name: `${task.name}/shard-${
|
||||||
String(index + 1).padStart(2, "0")
|
String(index + 1).padStart(2, "0")
|
||||||
}-of-${count}`,
|
}-of-${count}`,
|
||||||
args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames],
|
args: [
|
||||||
|
...task.args,
|
||||||
|
"--exact",
|
||||||
|
"--test-threads",
|
||||||
|
serial ? "1" : "2",
|
||||||
|
...selectedNames,
|
||||||
|
],
|
||||||
},
|
},
|
||||||
selectedNames,
|
selectedNames,
|
||||||
}));
|
}));
|
||||||
@@ -84,18 +103,27 @@ export function shardRustCommand(
|
|||||||
export async function discoverRustCommands(
|
export async function discoverRustCommands(
|
||||||
commands: TestCommand[],
|
commands: TestCommand[],
|
||||||
jobs: number,
|
jobs: number,
|
||||||
options: { output: string; signal?: AbortSignal },
|
options: {
|
||||||
|
output: string;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
run?: (task: TestCommand, label: string) => Promise<ExecutionResult>;
|
||||||
|
},
|
||||||
) {
|
) {
|
||||||
const binaries = await mapConcurrent(commands, jobs, async (task, index) => {
|
const binaries = await mapConcurrent(commands, jobs, async (task, index) => {
|
||||||
const listing = await execute({
|
const listingTask = {
|
||||||
...task,
|
...task,
|
||||||
name: `${task.name}/list`,
|
name: `${task.name}/list`,
|
||||||
args: [...task.args, "--list", "--format", "terse"],
|
args: [...task.args, "--list", "--format", "terse"],
|
||||||
}, {
|
};
|
||||||
...options,
|
const label = `list-${String(index).padStart(3, "0")}`;
|
||||||
label: `list-${String(index).padStart(3, "0")}`,
|
const listing = await (options.run
|
||||||
timeoutMs: SUITE_TIMEOUT_MS,
|
? options.run(listingTask, label)
|
||||||
});
|
: execute(listingTask, {
|
||||||
|
output: options.output,
|
||||||
|
signal: options.signal,
|
||||||
|
label,
|
||||||
|
timeoutMs: SUITE_TIMEOUT_MS,
|
||||||
|
}));
|
||||||
let shards: RustShard[] = [];
|
let shards: RustShard[] = [];
|
||||||
let error: string | undefined;
|
let error: string | undefined;
|
||||||
try {
|
try {
|
||||||
@@ -124,7 +152,8 @@ export async function discoverRustCommands(
|
|||||||
error,
|
error,
|
||||||
selectedCases: error
|
selectedCases: error
|
||||||
? null
|
? null
|
||||||
: shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0),
|
: shards.reduce((sum, shard) =>
|
||||||
|
sum + shard.selectedNames.length, 0),
|
||||||
shards,
|
shards,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
@@ -426,12 +455,149 @@ export interface TestCommand {
|
|||||||
command: string;
|
command: string;
|
||||||
args: string[];
|
args: string[];
|
||||||
cwd: string;
|
cwd: string;
|
||||||
env?: Record<string, string>;
|
env?: Record<string, string | undefined>;
|
||||||
// Exact link from each execution result into inventory.json, even when two
|
// Exact link from each execution result into inventory.json, even when two
|
||||||
// Cargo targets happen to share the same package/target display name.
|
// Cargo targets happen to share the same package/target display name.
|
||||||
inventoryRef?: { binaryIndex: number; shardIndex: number };
|
inventoryRef?: { binaryIndex: number; shardIndex: number };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Walk with lstat before creating any child: recursive mkdir/realPath alone
|
||||||
|
// would follow an existing symlink first. Reject aliases rather than silently
|
||||||
|
// canonicalizing a caller's checkout. As with the Rust guard, this assumes the
|
||||||
|
// owned checkout is not concurrently replaced by an adversarial filesystem actor.
|
||||||
|
async function ownedTestDirectory(checkout: string, relative: string) {
|
||||||
|
if (
|
||||||
|
!posix.isAbsolute(checkout) || checkout === "/" ||
|
||||||
|
posix.normalize(checkout) !== checkout
|
||||||
|
) throw new Error("test checkout must be an absolute normal-only path");
|
||||||
|
|
||||||
|
async function checkDirectory(path: string, owned: boolean) {
|
||||||
|
const stat = await Deno.lstat(path);
|
||||||
|
if (!stat.isDirectory || stat.isSymlink) {
|
||||||
|
throw new Error(
|
||||||
|
`test isolation refuses non-directory or symlink: ${path}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (owned && process.getuid && stat.uid !== process.getuid()) {
|
||||||
|
throw new Error(
|
||||||
|
`test isolation directory is not owned by this user: ${path}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (await Deno.realPath(path) !== path) {
|
||||||
|
throw new Error(
|
||||||
|
`test isolation refuses non-canonical directory: ${path}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let checkoutPath = "";
|
||||||
|
for (const part of checkout.slice(1).split("/")) {
|
||||||
|
checkoutPath += `/${part}`;
|
||||||
|
await checkDirectory(checkoutPath, false);
|
||||||
|
}
|
||||||
|
let path = checkout;
|
||||||
|
for (const part of relative.split("/")) {
|
||||||
|
if (!part || part === "." || part === "..") {
|
||||||
|
throw new Error("invalid owned test directory component");
|
||||||
|
}
|
||||||
|
path += `/${part}`;
|
||||||
|
try {
|
||||||
|
await Deno.mkdir(path, { mode: 0o700 });
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Deno.errors.AlreadyExists)) throw error;
|
||||||
|
}
|
||||||
|
await checkDirectory(path, true);
|
||||||
|
}
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Prepare child-only isolation for the maintained normal-test entrypoint.
|
||||||
|
* Each wrap gets a fresh allocation table, even for repeated/sanitized labels or
|
||||||
|
* concurrent runners using the same output id. Resources stay under target for
|
||||||
|
* diagnosis; callers with temporary checkouts own their recursive cleanup.
|
||||||
|
*/
|
||||||
|
export async function prepareTestIsolation(checkout: string, runId: string) {
|
||||||
|
const ownedDirectory = (relative: string) =>
|
||||||
|
ownedTestDirectory(checkout, relative);
|
||||||
|
const prefix = "target/restore-test-environment";
|
||||||
|
await ownedDirectory(prefix);
|
||||||
|
// Keep tempfile Unix-socket references short. tempfile owns unique resources
|
||||||
|
// here; neither runners nor suites remove/replace this shared directory.
|
||||||
|
const tmp = await ownedDirectory("target/t");
|
||||||
|
const digest = new Uint8Array(
|
||||||
|
await crypto.subtle.digest("SHA-256", new TextEncoder().encode(runId)),
|
||||||
|
);
|
||||||
|
const id = Array.from(
|
||||||
|
digest.slice(0, 6),
|
||||||
|
(byte) => byte.toString(16).padStart(2, "0"),
|
||||||
|
).join("");
|
||||||
|
const relativeRoot = `${prefix}/runner-${id}-${crypto.randomUUID()}`;
|
||||||
|
const root = await ownedDirectory(relativeRoot);
|
||||||
|
let next = 0;
|
||||||
|
return {
|
||||||
|
root,
|
||||||
|
async wrapCommand(task: TestCommand, label: string): Promise<TestCommand> {
|
||||||
|
const safeLabel = label.replaceAll(/[^a-zA-Z0-9_-]/g, "-").slice(0, 40) ||
|
||||||
|
"command";
|
||||||
|
const relative = `${relativeRoot}/${safeLabel}-${next++}`;
|
||||||
|
// Recheck shared ancestors as well as the command namespace on every wrap.
|
||||||
|
await ownedDirectory("target/t");
|
||||||
|
const namespace = await ownedDirectory(relative);
|
||||||
|
for (
|
||||||
|
const dir of [
|
||||||
|
"config",
|
||||||
|
"data",
|
||||||
|
"run",
|
||||||
|
"home",
|
||||||
|
"git-template/info",
|
||||||
|
"git-template/hooks",
|
||||||
|
]
|
||||||
|
) {
|
||||||
|
await ownedDirectory(`${relative}/${dir}`);
|
||||||
|
}
|
||||||
|
const inherited = { ...process.env, ...task.env };
|
||||||
|
const env: NonNullable<TestCommand["env"]> = { ...task.env };
|
||||||
|
// Undefined shadows inherited entries in execute's merge; node spawn omits
|
||||||
|
// them. Strip indexed Git config and less common Git path overrides too.
|
||||||
|
for (const key of Object.keys(inherited)) {
|
||||||
|
if (/^GIT_/i.test(key) || /(^|_)proxy$/i.test(key)) {
|
||||||
|
env[key] = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.assign(env, {
|
||||||
|
HOME: `${namespace}/home`,
|
||||||
|
YOI_HOME: `${namespace}/data`,
|
||||||
|
YOI_CONFIG_DIR: `${namespace}/config`,
|
||||||
|
YOI_DATA_DIR: `${namespace}/data`,
|
||||||
|
YOI_RUNTIME_DIR: `${namespace}/run`,
|
||||||
|
TMPDIR: tmp,
|
||||||
|
TMP: tmp,
|
||||||
|
TEMP: tmp,
|
||||||
|
GIT_TEMPLATE_DIR: `${namespace}/git-template`,
|
||||||
|
GIT_CONFIG_GLOBAL: "/dev/null",
|
||||||
|
GIT_CONFIG_SYSTEM: "/dev/null",
|
||||||
|
GIT_CONFIG_NOSYSTEM: "1",
|
||||||
|
GIT_ATTR_NOSYSTEM: "1",
|
||||||
|
GIT_CEILING_DIRECTORIES: checkout,
|
||||||
|
GIT_AUTHOR_NAME: "Normal test runner",
|
||||||
|
GIT_AUTHOR_EMAIL: "normal-tests@example.invalid",
|
||||||
|
GIT_COMMITTER_NAME: "Normal test runner",
|
||||||
|
GIT_COMMITTER_EMAIL: "normal-tests@example.invalid",
|
||||||
|
RUSTC_WRAPPER: "",
|
||||||
|
RUSTC_WORKSPACE_WRAPPER: "",
|
||||||
|
SCCACHE_DISABLE: "1",
|
||||||
|
});
|
||||||
|
// HOME is isolated, but toolchain/cache discovery must keep its ordinary
|
||||||
|
// host locations (these are compiler settings, not live product state).
|
||||||
|
for (const key of ["CARGO_HOME", "RUSTUP_HOME"] as const) {
|
||||||
|
const fallback = key === "CARGO_HOME" ? ".cargo" : ".rustup";
|
||||||
|
env[key] = inherited[key] ||
|
||||||
|
(process.env.HOME ? `${process.env.HOME}/${fallback}` : undefined);
|
||||||
|
}
|
||||||
|
return { ...task, env };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function rustTestCommands(
|
export function rustTestCommands(
|
||||||
artifacts: CargoArtifact[],
|
artifacts: CargoArtifact[],
|
||||||
packages: { id: string; name: string; manifest_path: string }[],
|
packages: { id: string; name: string; manifest_path: string }[],
|
||||||
@@ -497,16 +663,26 @@ async function main() {
|
|||||||
"",
|
"",
|
||||||
);
|
);
|
||||||
const jobs = parseJobs(Deno.args);
|
const jobs = parseJobs(Deno.args);
|
||||||
const output = `${root}/target/normal-tests/${
|
const outputId = `${
|
||||||
new Date().toISOString().replaceAll(":", "-")
|
new Date().toISOString().replaceAll(":", "-")
|
||||||
}`;
|
}-${crypto.randomUUID()}`;
|
||||||
await Deno.mkdir(output, { recursive: true });
|
const isolation = await prepareTestIsolation(root, outputId);
|
||||||
|
const output = await ownedTestDirectory(
|
||||||
|
root,
|
||||||
|
`target/normal-tests/${outputId}`,
|
||||||
|
);
|
||||||
const cancellation = installCancellationHandlers();
|
const cancellation = installCancellationHandlers();
|
||||||
const run = (
|
const run = async (
|
||||||
task: TestCommand,
|
task: TestCommand,
|
||||||
label: string,
|
label: string,
|
||||||
timeoutMs = SUITE_TIMEOUT_MS,
|
timeoutMs = SUITE_TIMEOUT_MS,
|
||||||
) => execute(task, { output, label, timeoutMs, signal: cancellation.signal });
|
) =>
|
||||||
|
execute(await isolation.wrapCommand(task, label), {
|
||||||
|
output,
|
||||||
|
label,
|
||||||
|
timeoutMs,
|
||||||
|
signal: cancellation.signal,
|
||||||
|
});
|
||||||
const summary: Record<string, unknown> = { phase: "metadata", failed: false };
|
const summary: Record<string, unknown> = { phase: "metadata", failed: false };
|
||||||
let executionStart: number | undefined;
|
let executionStart: number | undefined;
|
||||||
const writeSummary = () =>
|
const writeSummary = () =>
|
||||||
@@ -516,7 +692,7 @@ async function main() {
|
|||||||
{
|
{
|
||||||
jobs,
|
jobs,
|
||||||
concurrency:
|
concurrency:
|
||||||
"top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism",
|
"top-level suites/shards; worker, worker-runtime and yoi-workspace-server use 1 test thread; other large Rust shards use 2, other suites retain internal parallelism",
|
||||||
platform: "linux",
|
platform: "linux",
|
||||||
suiteTimeoutMs: SUITE_TIMEOUT_MS,
|
suiteTimeoutMs: SUITE_TIMEOUT_MS,
|
||||||
buildTimeoutMs: BUILD_TIMEOUT_MS,
|
buildTimeoutMs: BUILD_TIMEOUT_MS,
|
||||||
@@ -600,6 +776,7 @@ async function main() {
|
|||||||
const discovery = await discoverRustCommands(rust, jobs, {
|
const discovery = await discoverRustCommands(rust, jobs, {
|
||||||
output,
|
output,
|
||||||
signal: cancellation.signal,
|
signal: cancellation.signal,
|
||||||
|
run,
|
||||||
});
|
});
|
||||||
summary.discoverySeconds = (performance.now() - discoveryStart) / 1000;
|
summary.discoverySeconds = (performance.now() - discoveryStart) / 1000;
|
||||||
summary.inventory = "inventory.json";
|
summary.inventory = "inventory.json";
|
||||||
@@ -655,6 +832,7 @@ async function main() {
|
|||||||
"--allow-write",
|
"--allow-write",
|
||||||
"--allow-run",
|
"--allow-run",
|
||||||
"--allow-env",
|
"--allow-env",
|
||||||
|
"--allow-sys=uid",
|
||||||
"tools/test.test.ts",
|
"tools/test.test.ts",
|
||||||
"tools/check-state-contract-names.test.ts",
|
"tools/check-state-contract-names.test.ts",
|
||||||
],
|
],
|
||||||
@@ -663,7 +841,7 @@ async function main() {
|
|||||||
...discovery.tasks,
|
...discovery.tasks,
|
||||||
];
|
];
|
||||||
console.log(
|
console.log(
|
||||||
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
|
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (worker/worker-runtime/yoi-workspace-server: 1 test thread; other large Rust shards: 2; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
|
||||||
);
|
);
|
||||||
const results: ReturnType<typeof resultSummary>[] = [];
|
const results: ReturnType<typeof resultSummary>[] = [];
|
||||||
summary.results = results;
|
summary.results = results;
|
||||||
|
|||||||
Reference in New Issue
Block a user