Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat(worker): separate restored candidates, repair, and activation

Browse Source
This commit is contained in:
Hare
2026-10-22 22:58:57 +09:00
parent b5c105545c
commit 73a3e674c9
26 changed files with 6592 additions and 787 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/lib.rs
+3
View File
@@ -19,6 +19,9 @@ pub mod repositories;
pub mod repository_access;
pub mod repository_source;
pub mod resource_broker;
#[cfg(test)]
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
pub mod retention;
pub mod runtime_settings;
pub mod runtime_subscription;
crates/workspace-server/src/server/tests/restored_parent_control_tests.rs
+84 -6
View File
@@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{
WorkerRuntimeExecutionBackend,
};
pub(super) fn connected_restore_tempdir() -> tempfile::TempDir {
crate::restore_test_environment::require_restore_test_environment();
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap();
tempfile::Builder::new()
.prefix("c")
.rand_bytes(3)
.tempdir_in(checkout.join("target"))
.unwrap()
}
const WATCHDOG: Duration = Duration::from_secs(45);
#[derive(Clone)]
@@ -82,27 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory {
) -> std::result::Result<(), String> {
self.inner.preflight_restore(request).await
}
async fn restore_controller(
async fn recover_restore_source(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
self.inner.recover_restore_source(request).await
}
async fn prepare_restore(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let worker_ref = request.worker_ref.clone();
let inner = self.inner.prepare_restore(request).await?;
Ok(Box::new(CapturingCandidate {
inner,
worker_ref,
captures: self.captures.clone(),
}))
}
async fn cleanup_failed_restore(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.cleanup_failed_restore(request).await
}
async fn reconcile_stopped_worker(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
) -> std::result::Result<(), String> {
self.inner.reconcile_stopped_worker(worker_ref).await
}
fn activate_restored_controller(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
workspace_id: Option<&str>,
handle: &worker::WorkerHandle,
) {
self.inner
.activate_restored_controller(worker_ref, workspace_id, handle);
}
}
struct CapturingCandidate {
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_ref: worker_runtime::identity::WorkerRef,
captures: Arc<Captures>,
}
#[async_trait::async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
self.inner.initial_state()
}
async fn activate(
&mut self,
) -> std::result::Result<
RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let worker_ref = request.worker_ref.clone();
let controller = self.inner.restore_controller(request).await?;
let controller = self.inner.activate().await?;
self.captures
.handles
.lock()
.unwrap()
.push((worker_ref.clone(), controller.handle.clone()));
.push((self.worker_ref.clone(), controller.handle.clone()));
self.captures
.restored_clients
.lock()
.unwrap()
.push((worker_ref, controller.workspace_client.clone()));
.push((self.worker_ref.clone(), controller.workspace_client.clone()));
Ok(controller)
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.discard().await
}
}
struct Resources {
@@ -262,7 +340,7 @@ async fn call(tools: &BTreeMap<String, Arc<dyn Tool>>, name: &str, args: Value)
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
let root = tempfile::tempdir().unwrap();
let root = connected_restore_tempdir();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let mut config = test_server_config(root.path());
crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs
+1579 -1
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/server_assignment_attention_tests.rs
+50 -10
View File
@@ -490,21 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
self.controller(request.worker_ref.worker_id.to_string(), false)
.await
}
async fn restore_controller(
async fn prepare_restore(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
Ok(Box::new(AttentionRestoreCandidate {
inputs: Some((
AttentionWorkerFactory {
root: self.root.clone(),
},
request.worker_ref.worker_id.to_string(),
)),
}))
}
}
// Preparation owns only fixture inputs; reconstruction and Controller startup
// remain lazy so discard never needs to shut down a started Worker.
struct AttentionRestoreCandidate {
inputs: Option<(AttentionWorkerFactory, String)>,
}
#[async_trait::async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
protocol::WorkerStateSnapshot::initial()
}
async fn activate(
&mut self,
) -> std::result::Result<
worker_runtime::worker_backend::RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
self.controller(request.worker_ref.worker_id.to_string(), true)
.await
.map_err(|_| {
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
"attention fixture reconstruction failed",
)
})
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
"attention fixture candidate already consumed",
)
})?;
factory.controller(worker_id, true).await.map_err(|_| {
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
"attention fixture reconstruction failed",
)
})
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inputs.take();
Ok(())
}
}
@@ -611,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() {
async fn real_attention_worker_start_restore(lose_response: bool) {
use session_store::{LogEntry, Store, WorkerSessionStore};
let dir = tempfile::tempdir().unwrap();
let dir = restored_parent_control_tests::connected_restore_tempdir();
init_clean_git_workspace(dir.path());
let config = test_server_config(dir.path());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
crates/workspace-server/src/server_assignment_delivery_tests.rs
+115 -27
View File
@@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() {
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str());
let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
assert_eq!(page["items"].as_array().unwrap().len(),1);
assert_eq!(page["items"][0]["model_processing_confirmed"],false);
let path = format!(
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",
ticket.id.as_str()
);
let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
assert_eq!(page["items"].as_array().unwrap().len(), 1);
assert_eq!(page["items"][0]["model_processing_confirmed"], false);
assert!(page["items"][0].get("content").is_none());
let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
assert_eq!(again,page);
let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap();
assert_eq!(response.status(),StatusCode::UNAUTHORIZED);
delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await;
delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await;
let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
assert_eq!(again, page);
let response = build_router(api.clone())
.oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
delivery_http(
&api,
"GET",
"/api/w/other-workspace/notification-diagnostics",
None,
StatusCode::NOT_FOUND,
)
.await;
delivery_http(
&api,
"GET",
&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),
None,
StatusCode::BAD_REQUEST,
)
.await;
}
fn delivery_assignment_path(ticket: &str) -> String {
@@ -828,21 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
self.controller(request.worker_ref.worker_id.to_string(), false)
.await
}
async fn restore_controller(
async fn prepare_restore(
&self,
request: delivery_execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
Ok(Box::new(DeliveryRestoreCandidate {
inputs: Some((
DeliveryWorkerFactory {
root: self.root.clone(),
client: self.client.clone(),
},
request.worker_ref.worker_id.to_string(),
)),
}))
}
}
// Retain the same scripted client/receipt store without opening either store or
// starting a Controller until activation. Discard drops only these owned inputs.
struct DeliveryRestoreCandidate {
inputs: Option<(DeliveryWorkerFactory, String)>,
}
#[async_trait::async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
protocol::WorkerStateSnapshot::initial()
}
async fn activate(
&mut self,
) -> std::result::Result<
worker_runtime::worker_backend::RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
self.controller(request.worker_ref.worker_id.to_string(), true)
.await
.map_err(|_| {
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
"delivery fixture reconstruction failed",
)
})
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
"delivery fixture candidate already consumed",
)
})?;
factory.controller(worker_id, true).await.map_err(|_| {
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
"delivery fixture reconstruction failed",
)
})
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inputs.take();
Ok(())
}
}
@@ -941,6 +1002,7 @@ async fn delivery_real_fixture(
Arc<DeliveryResponseLossBackend>,
DeliveryScriptedClient,
) {
let _environment = restored_parent_control_tests::connected_restore_tempdir();
let config = test_server_config(root);
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
seed_test_registered_workspace(store.as_ref(), &config)
@@ -1146,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
"uncertain delivery changed request id or payload: {attempts:?}"
);
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery {
assignment_id: Some(assignment.into()), ..Default::default()
}).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt");
let unknown = api
.store
.notification_diagnostics(
TEST_WORKSPACE_ID,
&server_api::NotificationDiagnosticsQuery {
assignment_id: Some(assignment.into()),
..Default::default()
},
)
.unwrap()
.items
.into_iter()
.find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown)
.expect(
"saved unknown acceptance must remain visible while Runtime already has its receipt",
);
assert!(!unknown.model_processing_confirmed);
// Change mutable rendering context while the acknowledgement is unknown.
// A new self-report is not a new input and cannot change the frozen retry.
@@ -1209,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
}
delivery_wait_idle(&reopened, &worker, 2).await;
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await;
assert_eq!(evidence["items"][0]["stage"],"runtime_accepted");
assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id));
assert_eq!(evidence["items"][0]["model_processing_confirmed"],false);
assert_eq!(evidence["items"][0]["retry_candidate"],false);
let evidence = delivery_http(
&reopened,
"GET",
&format!(
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",
unknown.source_id
),
None,
StatusCode::OK,
)
.await;
assert_eq!(evidence["items"][0]["stage"], "runtime_accepted");
assert_eq!(
evidence["items"][0]["delivery_id"],
json!(unknown.delivery_id)
);
assert_eq!(evidence["items"][0]["model_processing_confirmed"], false);
assert_eq!(evidence["items"][0]["retry_candidate"], false);
assert_eq!(
delivery_commits(&reopened, &worker)
.iter()
crates/workspace-server/src/server_workspace_config_wip_tests.rs
+85 -4
View File
@@ -187,7 +187,14 @@ async fn config_call(
runtime
.invoke(
path.into(),
if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)},
if matches!(operation, "edit" | "write") {
wip_protocol::InterfaceReference {
scope: "/".into(),
name: format!("yoi.file/{operation}-text/v1"),
}
} else {
config_interface(path)
},
operation.into(),
args,
agen::tool::ToolExecutionContext::direct(),
@@ -581,14 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
Ok(controller)
}
async fn restore_controller(
async fn preflight_restore(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<(), String> {
self.inner.preflight_restore(request).await
}
async fn recover_restore_source(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
self.inner.recover_restore_source(request).await
}
async fn prepare_restore(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let inner = self.inner.prepare_restore(request).await?;
Ok(Box::new(ConfigCaptureCandidate {
inner,
client: self.client.clone(),
controller_abort: self.controller_abort.clone(),
}))
}
async fn cleanup_failed_restore(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.cleanup_failed_restore(request).await
}
async fn reconcile_stopped_worker(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
) -> std::result::Result<(), String> {
self.inner.reconcile_stopped_worker(worker_ref).await
}
fn activate_restored_controller(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
workspace_id: Option<&str>,
handle: &worker::WorkerHandle,
) {
self.inner
.activate_restored_controller(worker_ref, workspace_id, handle);
}
}
struct ConfigCaptureCandidate {
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
client: Arc<Mutex<Option<Arc<dyn worker::WorkspaceClient>>>>,
controller_abort: Arc<Mutex<Option<tokio::task::AbortHandle>>>,
}
#[async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
self.inner.initial_state()
}
async fn activate(
&mut self,
) -> std::result::Result<
worker_runtime::worker_backend::RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
self.inner.restore_controller(request).await
let controller = self.inner.activate().await?;
*self.client.lock().unwrap() = Some(controller.workspace_client.clone());
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
Ok(controller)
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.discard().await
}
}
@@ -717,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
decode_runtime_request_source_claims,
};
let root = restored_parent_control_tests::connected_restore_tempdir();
// Remote Runtime-owned HTTP adapter, including its real request signer.
let mut fixture = manual_worker_assignment_fixture().await;
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
@@ -796,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
// Actual embedded Runtime factory -> real Worker controller -> captured
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
let root = tempfile::tempdir().unwrap();
init_clean_git_workspace(root.path());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();